Addon: bundle BAT and start of interfacing with it

The add-on can now create BAT packs, but still only at a hard-coded path.

addon/flamenco/__init__.py

@@ -35,6 +35,12 @@ def discard_global_flamenco_data(_):
     comms.discard_flamenco_data()
 
 
+def redraw(self, context):
+    if context.area is None:
+        return
+    context.area.tag_redraw()
+
+
 def register() -> None:
     from . import dependencies
 
@@ -43,6 +49,46 @@ def register() -> None:
     bpy.app.handlers.load_pre.append(discard_global_flamenco_data)
     bpy.app.handlers.load_factory_preferences_post.append(discard_global_flamenco_data)
 
+    bpy.types.WindowManager.flamenco_bat_status = bpy.props.EnumProperty(
+        items=[
+            ("IDLE", "IDLE", "Not doing anything."),
+            ("SAVING", "SAVING", "Saving your file."),
+            ("INVESTIGATING", "INVESTIGATING", "Finding all dependencies."),
+            ("TRANSFERRING", "TRANSFERRING", "Transferring all dependencies."),
+            ("COMMUNICATING", "COMMUNICATING", "Communicating with Flamenco Server."),
+            ("DONE", "DONE", "Not doing anything, but doing something earlier."),
+            ("ABORTING", "ABORTING", "User requested we stop doing something."),
+            ("ABORTED", "ABORTED", "We stopped doing something."),
+        ],
+        name="flamenco_status",
+        default="IDLE",
+        description="Current status of the Flamenco add-on",
+        update=redraw,
+    )
+
+    bpy.types.WindowManager.flamenco_bat_status_txt = bpy.props.StringProperty(
+        name="Flamenco Status",
+        default="",
+        description="Textual description of what Flamenco is doing",
+        update=redraw,
+    )
+
+    bpy.types.WindowManager.flamenco_bat_progress = bpy.props.IntProperty(
+        name="Flamenco Progress",
+        default=0,
+        description="File transfer progress",
+        subtype="PERCENTAGE",
+        min=0,
+        max=100,
+        update=redraw,
+    )
+
+    bpy.types.Scene.flamenco_job_name = bpy.props.StringProperty(
+        name="Flamenco Job Name",
+        default="",
+        description="Name of the Flamenco job; an empty name will use the blend file name as job name",
+    )
+
     # Placeholder to contain the result of a 'ping' to Flamenco Manager,
     # so that it can be shown in the preferences panel.
     bpy.types.WindowManager.flamenco_status_ping = bpy.props.StringProperty()

addon/flamenco/bat_interface.py

@@ -0,0 +1,283 @@
+"""BAT packing interface for Flamenco."""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+import logging
+import queue
+import threading
+import typing
+
+import bpy
+
+from . import wheels
+
+pack = wheels.load_wheel("blender_asset_tracer.pack")
+progress = wheels.load_wheel("blender_asset_tracer.pack.progress")
+transfer = wheels.load_wheel("blender_asset_tracer.pack.transfer")
+shaman = wheels.load_wheel("blender_asset_tracer.pack.shaman")
+
+log = logging.getLogger(__name__)
+
+# For using in other parts of the add-on, so only this file imports BAT.
+Aborted = pack.Aborted
+FileTransferError = transfer.FileTransferError
+parse_shaman_endpoint = shaman.parse_endpoint
+
+
+class Message:
+    """Superclass for message objects queued by the BatProgress class."""
+
+
+@dataclass
+class MsgSetWMAttribute(Message):
+    """Set a WindowManager attribute to a value."""
+
+    attribute_name: str
+    value: object
+
+
+@dataclass
+class MsgException(Message):
+    """Report an exception."""
+
+    ex: BaseException
+
+
+@dataclass
+class MsgProgress(Message):
+    """Report packing progress."""
+
+    percentage: int  # 0 - 100
+
+
+@dataclass
+class MsgDone(Message):
+    output_path: Path
+    missing_files: list[Path]
+
+
+class BatProgress(progress.Callback):
+    """Report progress of BAT Packing to the given queue."""
+
+    def __init__(self, queue: queue.Queue[Message]) -> None:
+        super().__init__()
+        self.queue = queue
+
+    def _set_attr(self, attr: str, value):
+        msg = MsgSetWMAttribute(attr, value)
+        self.queue.put(msg)
+
+    def _txt(self, msg: str):
+        """Set a text in a thread-safe way."""
+        self._set_attr("flamenco_bat_status_txt", msg)
+
+    def _status(self, status: str):
+        """Set the flamenco_bat_status property in a thread-safe way."""
+        self._set_attr("flamenco_bat_status", status)
+
+    def _progress(self, progress: int):
+        """Set the flamenco_bat_progress property in a thread-safe way."""
+        self._set_attr("flamenco_bat_progress", progress)
+        msg = MsgProgress(percentage=progress)
+        self.queue.put(msg)
+
+    def pack_start(self) -> None:
+        self._txt("Starting BAT Pack operation")
+
+    def pack_done(
+        self, output_blendfile: Path, missing_files: typing.Set[Path]
+    ) -> None:
+        if missing_files:
+            self._txt("There were %d missing files" % len(missing_files))
+        else:
+            self._txt("Pack of %s done" % output_blendfile.name)
+
+    def pack_aborted(self, reason: str):
+        self._txt("Aborted: %s" % reason)
+        self._status("ABORTED")
+
+    def trace_blendfile(self, filename: Path) -> None:
+        """Called for every blendfile opened when tracing dependencies."""
+        self._txt("Inspecting %s" % filename.name)
+
+    def trace_asset(self, filename: Path) -> None:
+        if filename.stem == ".blend":
+            return
+        self._txt("Found asset %s" % filename.name)
+
+    def rewrite_blendfile(self, orig_filename: Path) -> None:
+        self._txt("Rewriting %s" % orig_filename.name)
+
+    def transfer_file(self, src: Path, dst: Path) -> None:
+        self._txt("Transferring %s" % src.name)
+
+    def transfer_file_skipped(self, src: Path, dst: Path) -> None:
+        self._txt("Skipped %s" % src.name)
+
+    def transfer_progress(self, total_bytes: int, transferred_bytes: int) -> None:
+        self._progress(round(100 * transferred_bytes / total_bytes))
+
+    def missing_file(self, filename: Path) -> None:
+        # TODO(Sybren): report missing files in a nice way
+        pass
+
+
+# class ShamanPacker(shaman.ShamanPacker):
+#     """Packer with support for getting an auth token from Flamenco Server."""
+
+#     def __init__(
+#         self,
+#         bfile: Path,
+#         project: Path,
+#         target: str,
+#         endpoint: str,
+#         checkout_id: str,
+#         *,
+#         manager_id: str,
+#         **kwargs
+#     ) -> None:
+#         self.manager_id = manager_id
+#         super().__init__(bfile, project, target, endpoint, checkout_id, **kwargs)
+
+#     def _get_auth_token(self) -> str:
+#         """get a token from Flamenco Server"""
+
+#         from ..blender import PILLAR_SERVER_URL
+#         from ..pillar import blender_id_subclient, uncached_session, SUBCLIENT_ID
+
+#         url = urllib.parse.urljoin(
+#             PILLAR_SERVER_URL, "flamenco/jwt/generate-token/%s" % self.manager_id
+#         )
+#         auth_token = blender_id_subclient()["token"]
+
+#         resp = uncached_session.get(url, auth=(auth_token, SUBCLIENT_ID))
+#         resp.raise_for_status()
+#         return resp.text
+
+
+class PackThread(threading.Thread):
+    queue: queue.Queue[Message]
+
+    def __init__(self, packer: pack.Packer) -> None:
+        # Quitting Blender should abort the transfer (instead of hanging until
+        # the transfer is done), hence daemon=True.
+        super().__init__(daemon=True, name="PackThread")
+
+        self.queue = queue.SimpleQueue()
+
+        self.packer = packer
+        self.packer.progress_cb = BatProgress(queue=self.queue)
+
+    def run(self) -> None:
+        global _running_packthread
+
+        try:
+            self._run()
+        except BaseException as ex:
+            log.error("Error packing with BAT: %s", ex)
+            self.queue.put(MsgException(ex=ex))
+        finally:
+            with _packer_lock:
+                _running_packthread = None
+
+    def _run(self) -> None:
+        with self.packer:
+            log.debug("awaiting strategise")
+            self._set_bat_status("INVESTIGATING")
+            self.packer.strategise()
+
+            log.debug("awaiting execute")
+            self._set_bat_status("TRANSFERRING")
+            self.packer.execute()
+
+            log.debug("done")
+            self._set_bat_status("DONE")
+
+            msg = MsgDone(self.packer.output_path, self.packer.missing_files)
+            self.queue.put(msg)
+
+    def _set_bat_status(self, status: str) -> None:
+        self.queue.put(MsgSetWMAttribute("flamenco_bat_status", status))
+
+    def poll(self, timeout: Optional[int] = None) -> Optional[Message]:
+        """Poll the queue, return the first message or None if there is none.
+
+        :param timeout: Max time to wait for a message to appear on the queue.
+            If None, will not wait and just return None immediately (if there is
+            no queued message).
+        """
+        try:
+            return self.queue.get(block=timeout is not None, timeout=timeout)
+        except queue.Empty:
+            return None
+
+    def abort(self) -> None:
+        """Abort the running pack operation."""
+        self.packer.abort()
+
+
+_running_packthread: typing.Optional[PackThread] = None
+_packer_lock = threading.RLock()
+
+
+def copy(
+    base_blendfile: Path,
+    project: Path,
+    target: str,
+    exclusion_filter: str,
+    *,
+    relative_only: bool,
+    packer_class=pack.Packer,
+    **packer_args
+) -> PackThread:
+    """Use BAT to copy the given file and dependencies to the target location.
+
+    Runs BAT in a separate thread, and returns early. Use poll() to get updates
+    & the final result.
+    """
+    global _running_packthread
+
+    with _packer_lock:
+        if _running_packthread is not None:
+            raise RuntimeError("other packing operation already in progress")
+
+    packer = packer_class(
+        base_blendfile,
+        project,
+        target,
+        compress=True,
+        relative_only=relative_only,
+        **packer_args
+    )
+    if exclusion_filter:
+        filter_parts = exclusion_filter.strip().split(" ")
+        packer.exclude(*filter_parts)
+
+    packthread = PackThread(packer=packer)
+    with _packer_lock:
+        _running_packthread = packthread
+
+    packthread.start()
+    return packthread
+
+
+def abort() -> None:
+    """Abort a running copy() call.
+
+    No-op when there is no running copy(). Can be called from any thread.
+    """
+
+    with _packer_lock:
+        if _running_packthread is None:
+            log.debug("No running packer, ignoring call to abort()")
+            return
+        log.info("Aborting running packer")
+        _running_packthread.abort()
+
+
+def is_packing() -> bool:
+    """Returns whether a BAT packing operation is running."""
+
+    with _packer_lock:
+        return _running_packthread is not None

addon/flamenco/gui.py

@@ -25,6 +25,12 @@ class FLAMENCO_PT_job_submission(bpy.types.Panel):
         row.operator("flamenco.fetch_job_types", text="", icon="FILE_REFRESH")
         self.draw_job_settings(context, layout)
 
+        layout.separator()
+        col = layout.column(align=True)
+        col.prop(context.scene, "flamenco_job_name", text="Job Name")
+
+        self.draw_flamenco_status(context, layout)
+
     def draw_job_settings(
         self, context: bpy.types.Context, layout: bpy.types.UILayout
     ) -> None:
@@ -43,6 +49,42 @@ class FLAMENCO_PT_job_submission(bpy.types.Panel):
                 continue
             layout.prop(propgroup, setting.key)
 
+    def draw_flamenco_status(
+        self, context: bpy.types.Context, layout: bpy.types.UILayout
+    ) -> None:
+        # Show current status of Flamenco.
+        flamenco_status = context.window_manager.flamenco_bat_status
+        if flamenco_status in {"IDLE", "ABORTED", "DONE"}:
+            ui = layout
+            props = ui.operator(
+                "flamenco.submit_job",
+                text="Submit to Flamenco",
+                icon="RENDER_ANIMATION",
+            )
+            props.job_name = context.scene.flamenco_job_name
+        elif flamenco_status == "INVESTIGATING":
+            row = layout.row(align=True)
+            row.label(text="Investigating your files")
+            # row.operator(FLAMENCO_OT_abort.bl_idname, text="", icon="CANCEL")
+        elif flamenco_status == "COMMUNICATING":
+            layout.label(text="Communicating with Flamenco Server")
+        elif flamenco_status == "ABORTING":
+            row = layout.row(align=True)
+            row.label(text="Aborting, please wait.")
+            # row.operator(FLAMENCO_OT_abort.bl_idname, text="", icon="CANCEL")
+        if flamenco_status == "TRANSFERRING":
+            row = layout.row(align=True)
+            row.prop(
+                context.window_manager,
+                "flamenco_bat_progress",
+                text=context.window_manager.flamenco_bat_status_txt,
+            )
+            # row.operator(FLAMENCO_OT_abort.bl_idname, text="", icon="CANCEL")
+        elif (
+            flamenco_status != "IDLE" and context.window_manager.flamenco_bat_status_txt
+        ):
+            layout.label(text=context.window_manager.flamenco_bat_status_txt)
+
 
 classes = (FLAMENCO_PT_job_submission,)
 register, unregister = bpy.utils.register_classes_factory(classes)

addon/flamenco/operators.py

@@ -1,8 +1,22 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 # <pep8 compliant>
 
+import logging
+from pathlib import Path, PurePath
+from typing import Optional, TYPE_CHECKING
+
 import bpy
 
+from . import preferences
+
+if TYPE_CHECKING:
+    from .bat_interface import PackThread, Message
+else:
+    PackThread = object
+    Message = object
+
+_log = logging.getLogger(__name__)
+
 
 class FlamencoOpMixin:
     @staticmethod
@@ -59,6 +73,7 @@ class FLAMENCO_OT_ping_manager(FlamencoOpMixin, bpy.types.Operator):
         from flamenco.manager import ApiException
         from flamenco.manager.apis import MetaApi
         from flamenco.manager.models import FlamencoVersion
+        from urllib3.exceptions import HTTPError, MaxRetryError
 
         context.window_manager.flamenco_status_ping = "..."
 
@@ -68,6 +83,14 @@ class FLAMENCO_OT_ping_manager(FlamencoOpMixin, bpy.types.Operator):
         except ApiException as ex:
             report = "Manager cannot be reached: %s" % ex
             level = "ERROR"
+        except MaxRetryError as ex:
+            # This is the common error, when for example the port number is
+            # incorrect and nothing is listening.
+            report = "Manager cannot be reached"
+            level = "WARNING"
+        except HTTPError as ex:
+            report = "Manager cannot be reached: %s" % ex
+            level = "ERROR"
         else:
             report = "%s version %s found" % (response.name, response.version)
             level = "INFO"
@@ -77,8 +100,146 @@ class FLAMENCO_OT_ping_manager(FlamencoOpMixin, bpy.types.Operator):
         return {"FINISHED"}
 
 
+class FLAMENCO_OT_submit_job(FlamencoOpMixin, bpy.types.Operator):
+    bl_idname = "flamenco.submit_job"
+    bl_label = "Flamenco: Submit Job"
+    bl_description = "Pack the current blend file and send it to Flamenco"
+    bl_options = {"REGISTER"}  # No UNDO.
+
+    job_name: bpy.props.StringProperty(name="Job Name")
+
+    timer: Optional[bpy.types.Timer] = None
+    packthread: Optional[PackThread] = None
+
+    log = _log.getChild(bl_idname)
+
+    def invoke(self, context: bpy.types.Context, event) -> set[str]:
+        filepath = self._save_blendfile(context)
+        return self._bat_pack(context, filepath)
+
+    def modal(self, context: bpy.types.Context, event) -> set[str]:
+        # This function is called for TIMER events to poll the BAT pack thread.
+        if event.type != "TIMER":
+            return {"PASS_THROUGH"}
+
+        if self.packthread is None:
+            # If there is no pack thread running, there isn't much we can do.
+            return self._quit(context)
+
+        msg = self.packthread.poll()
+        if not msg:
+            return {"RUNNING_MODAL"}
+
+        return self._on_bat_pack_msg(context, msg)
+
+    def _save_blendfile(self, context):
+        """Save to a different file, specifically for Flamenco.
+
+        We shouldn't overwrite the artist's file.
+        We can compress, since this file won't be managed by SVN and doesn't need diffability.
+        """
+        render = context.scene.render
+
+        # Remember settings we need to restore after saving.
+        old_use_file_extension = render.use_file_extension
+        old_use_overwrite = render.use_overwrite
+        old_use_placeholder = render.use_placeholder
+
+        # TODO: see about disabling the denoiser (like the old Blender Cloud addon did).
+
+        try:
+            # The file extension should be determined by the render settings, not necessarily
+            # by the setttings in the output panel.
+            render.use_file_extension = True
+
+            # Rescheduling should not overwrite existing frames.
+            render.use_overwrite = False
+            render.use_placeholder = False
+
+            filepath = Path(context.blend_data.filepath).with_suffix(".flamenco.blend")
+            self.log.info("Saving copy to temporary file %s", filepath)
+            bpy.ops.wm.save_as_mainfile(
+                filepath=str(filepath), compress=True, copy=True
+            )
+        finally:
+            # Restore the settings we changed, even after an exception.
+            render.use_file_extension = old_use_file_extension
+            render.use_overwrite = old_use_overwrite
+            render.use_placeholder = old_use_placeholder
+
+        return filepath
+
+    def _bat_pack(self, context: bpy.types.Context, blendfile: Path) -> set[str]:
+        from . import bat_interface
+
+        if bat_interface.is_packing():
+            self.report({"ERROR"}, "Another packing operation is running")
+            self._quit()
+            return {"CANCELLED"}
+
+        # TODO: get project path from addon preferences / project definition on Manager.
+        project_path = blendfile.parent
+        try:
+            project_path = Path(bpy.path.abspath(str(project_path))).resolve()
+        except FileNotFoundError:
+            # Path.resolve() will raise a FileNotFoundError if the project path doesn't exist.
+            self.report({"ERROR"}, "Project path %s does not exist" % project_path)
+            return {"CANCELLED"}
+
+        # Determine where the render output will be stored.
+        render_output = Path("/render/_flamenco/tests/renders") / self.job_name
+        self.log.info("Will output render files to %s", render_output)
+
+        self.packthread = bat_interface.copy(
+            base_blendfile=blendfile,
+            project=project_path,
+            target=render_output,
+            exclusion_filter="",  # TODO: get from GUI.
+            relative_only=True,  # TODO: get from GUI.
+        )
+
+        context.window_manager.modal_handler_add(self)
+        wm = context.window_manager
+        self.timer = wm.event_timer_add(0.25, window=context.window)
+
+        return {"RUNNING_MODAL"}
+
+    def _on_bat_pack_msg(self, context: bpy.types.Context, msg: Message) -> set[str]:
+        from . import bat_interface
+
+        if isinstance(msg, bat_interface.MsgDone):
+            self.report({"INFO"}, "BAT pack is done")
+            # TODO: actually send the job to Flamenco!
+            return self._quit(context)
+
+        if isinstance(msg, bat_interface.MsgException):
+            self.log.error("Error performing BAT pack: %s", msg.ex)
+            self.report({"ERROR"}, "Error performing BAT pack")
+
+            # This was an exception caught at the top level of the thread, so
+            # the packing thread itself has stopped.
+            return self._quit(context)
+
+        if isinstance(msg, bat_interface.MsgSetWMAttribute):
+            wm = context.window_manager
+            setattr(wm, msg.attribute_name, msg.value)
+
+        return {"RUNNING_MODAL"}
+
+    def _quit(self, context: bpy.types.Context) -> set[str]:
+        """Stop any timer and return a 'FINISHED' status.
+
+        Does neither check nor abort the BAT pack thread.
+        """
+        if self.timer is not None:
+            context.window_manager.event_timer_remove(self.timer)
+            self.timer = None
+        return {"FINISHED"}
+
+
 classes = (
     FLAMENCO_OT_fetch_job_types,
     FLAMENCO_OT_ping_manager,
+    FLAMENCO_OT_submit_job,
 )
 register, unregister = bpy.utils.register_classes_factory(classes)

addon/flamenco/preferences.py

@@ -31,12 +31,18 @@ class FlamencoPreferences(bpy.types.AddonPreferences):
         col.label(text=context.window_manager.flamenco_status_ping)
 
 
-def manager_url(context: bpy.types.Context) -> str:
+def get(context: bpy.types.Context) -> FlamencoPreferences:
-    """Returns the configured Manager URL."""
+    """Return the add-on preferences."""
     prefs = context.preferences.addons["flamenco"].preferences
     assert isinstance(
         prefs, FlamencoPreferences
     ), "Expected FlamencoPreferences, got %s instead" % (type(prefs))
+    return prefs
+
+
+def manager_url(context: bpy.types.Context) -> str:
+    """Returns the configured Manager URL."""
+    prefs = get(context)
     return str(prefs.manager_url)