diff --git a/Makefile b/Makefile index 347db099..b19ad4bd 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,6 @@ generate-py: -i pkg/api/flamenco-manager.yaml \ -g python \ -o addon/ \ - --skip-validate-spec \ --package-name "${PY_API_PKG_NAME}" \ --http-user-agent "Flamenco/${VERSION} (Blender add-on)" \ -p generateSourceCodeOnly=true \ diff --git a/addon/flamenco/bat/interface.py b/addon/flamenco/bat/interface.py index 29c67d9d..e02fc022 100644 --- a/addon/flamenco/bat/interface.py +++ b/addon/flamenco/bat/interface.py @@ -197,7 +197,7 @@ def copy( # type: ignore *, relative_only: bool, packer_class=pack.Packer, - packer_kwargs: Optional[dict[Any, Any]] = None, + **packer_kwargs: dict[Any, Any], ) -> PackThread: """Use BAT to copy the given file and dependencies to the target location. @@ -210,17 +210,12 @@ def copy( # type: ignore if _running_packthread is not None: raise RuntimeError("other packing operation already in progress") - print(f"packer_class: {packer_class}") - if packer_kwargs is None: - packer_kwargs = {} - packer_kwargs.setdefault("compress", True) - packer_kwargs.setdefault("relative_only", relative_only) - print(f"packer_kwargs: {packer_kwargs}") - packer = packer_class( base_blendfile, project, target, + compress=True, + relative_only=relative_only, **packer_kwargs, ) if exclusion_filter: diff --git a/addon/flamenco/bat/shaman.py b/addon/flamenco/bat/shaman.py index c043ed38..8b1cf13b 100644 --- a/addon/flamenco/bat/shaman.py +++ b/addon/flamenco/bat/shaman.py @@ -5,7 +5,7 @@ import logging import random from collections import deque from pathlib import Path, PurePath, PurePosixPath -from typing import TYPE_CHECKING, Optional, Any +from typing import TYPE_CHECKING, Optional, Any, Iterable, Iterator from .. import wheels from . import cache @@ -34,6 +34,8 @@ log = logging.getLogger(__name__) MAX_DEFERRED_PATHS = 8 MAX_FAILED_PATHS = 8 +HashableShamanFileSpec = tuple[str, int, str] +"""Tuple of the 'sha', 'size', and 'path' fields of a ShamanFileSpec.""" # Mypy doesn't understand that bat_pack.Packer exists. class Packer(bat_pack.Packer): # type: ignore @@ -151,21 +153,21 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore def _upload_missing_files( self, shaman_file_specs: _ShamanRequirementsRequest - ) -> set[_ShamanFileSpec]: + ) -> list[_ShamanFileSpec]: self.log.info("Feeding %d files to the Shaman", len(shaman_file_specs.files)) if self.log.isEnabledFor(logging.INFO): for spec in shaman_file_specs.files: self.log.info(" - %s", spec.path) # Try to upload all the files. - failed_files: set[_ShamanFileSpec] = set() + failed_files: set[HashableShamanFileSpec] = set() max_tries = 50 for try_index in range(max_tries): # Send the file to the Shaman and see what we still need to send there. to_upload = self._send_checkout_def_to_shaman(shaman_file_specs) if to_upload is None: # An error has already been logged. - return failed_files + return make_file_specs_regular_list(failed_files) if not to_upload: break @@ -180,7 +182,7 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore # clients are sending the same files. Instead of retrying on a # file-by-file basis, we just re-send the checkout definition # file to the Shaman and obtain a new list of files to upload. - return failed_files + return make_file_specs_regular_list(failed_files) def _create_checkout_definition(self) -> Optional[_ShamanRequirementsRequest]: """Create the checkout definition file for this BAT pack. @@ -241,6 +243,7 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore None if there was an error. """ from ..manager.exceptions import ApiException + from ..manager.models import ShamanRequirementsResponse try: resp = self.shaman_api.shaman_checkout_requirements(requirements) @@ -250,11 +253,10 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore self.log.error(msg) self.error_set(msg) return None - assert isinstance(resp, _ShamanRequirementsResponse) + assert isinstance(resp, ShamanRequirementsResponse) to_upload: deque[_ShamanFileSpec] = deque() for file_spec in resp.files: - print(file_spec) if file_spec.path not in self._rel_to_local_path: msg = ( "Shaman requested path we did not intend to upload: %r" % file_spec @@ -263,7 +265,7 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore self.error_set(msg) return None - self.log.debug(" %s: %s", file_spec.status.value, file_spec.path) + self.log.debug(" %s: %s", file_spec.status, file_spec.path) match file_spec.status.value: case "unknown": to_upload.appendleft(file_spec) @@ -276,7 +278,9 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore return None return to_upload - def _upload_files(self, to_upload: deque[_ShamanFileSpec]) -> set[_ShamanFileSpec]: + def _upload_files( + self, to_upload: deque[_ShamanFileSpec] + ) -> set[HashableShamanFileSpec]: """Actually upload the files to Shaman. Returns the set of files that we did not upload. @@ -288,8 +292,8 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore from ..manager.exceptions import ApiException - failed_specs: set[_ShamanFileSpec] = set() - deferred_specs: set[_ShamanFileSpec] = set() + failed_specs: set[HashableShamanFileSpec] = set() + deferred_specs: set[HashableShamanFileSpec] = set() def defer(filespec: _ShamanFileSpec) -> None: nonlocal to_upload @@ -297,7 +301,7 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore self.log.info( " %s deferred (already being uploaded by someone else)", filespec.path ) - deferred_specs.add(filespec) + deferred_specs.add(make_file_spec_hashable(filespec)) # Instead of deferring this one file, randomize the files to upload. # This prevents multiple deferrals when someone else is uploading @@ -307,13 +311,15 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore to_upload = deque(all_files) self.log.info( - "Going to upload %d of %d files", len(to_upload), len(self._spec_to_paths) + "Going to upload %d of %d files", + len(to_upload), + len(self._rel_to_local_path), ) while to_upload: # After too many failures, just retry to get a fresh set of files to upload. if len(failed_specs) > MAX_FAILED_PATHS: self.log.info("Too many failures, going to abort this iteration") - failed_specs.update(to_upload) + failed_specs.update(make_file_specs_hashable_gen(to_upload)) return failed_specs file_spec = to_upload.popleft() @@ -331,9 +337,10 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore continue # Let the Shaman know whether we can defer uploading this file or not. - can_defer = ( + hashable_file_spec = make_file_spec_hashable(file_spec) + can_defer = bool( len(deferred_specs) < MAX_DEFERRED_PATHS - and file_spec not in deferred_specs + and hashable_file_spec not in deferred_specs and len(to_upload) ) @@ -370,10 +377,10 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore self.log.error(msg) self.error_set(msg) - failed_specs.add(file_spec) + failed_specs.add(make_file_spec_hashable(file_spec)) return failed_specs - failed_specs.discard(file_spec) + failed_specs.discard(make_file_spec_hashable(file_spec)) self.uploaded_files += 1 file_size = local_filepath.stat().st_size self.uploaded_bytes += file_size @@ -411,12 +418,12 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore from ..manager.exceptions import ApiException self.log.info( - "Requesting checkout at Shaman for checkout_path=%r", self.checkout_path + "Requesting checkout at Shaman for checkout_path=%s", self.checkout_path ) checkoutRequest = ShamanCheckout( files=shaman_file_specs.files, - checkout_path=self.checkout_path, + checkout_path=str(self.checkout_path), ) try: @@ -440,3 +447,30 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore return self.log.info("Shaman created checkout at %s", self.checkout_path) + + +def make_file_spec_hashable(spec: _ShamanFileSpec) -> HashableShamanFileSpec: + """Return a hashable, immutable representation of the given spec.""" + return (spec.sha, spec.size, spec.path) + + +def make_file_spec_regular(hashable_spec: HashableShamanFileSpec) -> _ShamanFileSpec: + """Convert a hashable filespec into a real one.""" + from ..manager.models import ShamanFileSpec + + spec: ShamanFileSpec = ShamanFileSpec(*hashable_spec) + return spec + + +def make_file_specs_hashable_gen( + specs: Iterable[_ShamanFileSpec], +) -> Iterator[HashableShamanFileSpec]: + """Convert a collection of specifications by generating their hashable representations.""" + return (make_file_spec_hashable(spec) for spec in specs) + + +def make_file_specs_regular_list( + hashable_specs: Iterable[HashableShamanFileSpec], +) -> list[_ShamanFileSpec]: + """Convert hashable filespecs into a list of real ones.""" + return [make_file_spec_regular(spec) for spec in hashable_specs] diff --git a/addon/flamenco/manager/__init__.py b/addon/flamenco/manager/__init__.py index 7ab5e9ad..6b7f4cb7 100644 --- a/addon/flamenco/manager/__init__.py +++ b/addon/flamenco/manager/__init__.py @@ -10,7 +10,7 @@ """ -__version__ = "8a43c69f-dirty" +__version__ = "8a97cf50-dirty" # import ApiClient from flamenco.manager.api_client import ApiClient diff --git a/addon/flamenco/manager/api_client.py b/addon/flamenco/manager/api_client.py index 987229f7..289608dd 100644 --- a/addon/flamenco/manager/api_client.py +++ b/addon/flamenco/manager/api_client.py @@ -76,7 +76,7 @@ class ApiClient(object): self.default_headers[header_name] = header_value self.cookie = cookie # Set default User-Agent. - self.user_agent = 'Flamenco/8a43c69f-dirty (Blender add-on)' + self.user_agent = 'Flamenco/8a97cf50-dirty (Blender add-on)' def __enter__(self): return self diff --git a/addon/flamenco/manager/configuration.py b/addon/flamenco/manager/configuration.py index e3737a06..62a36c63 100644 --- a/addon/flamenco/manager/configuration.py +++ b/addon/flamenco/manager/configuration.py @@ -404,7 +404,7 @@ conf = flamenco.manager.Configuration( "OS: {env}\n"\ "Python Version: {pyversion}\n"\ "Version of the API: 1.0.0\n"\ - "SDK Package Version: 8a43c69f-dirty".\ + "SDK Package Version: 8a97cf50-dirty".\ format(env=sys.platform, pyversion=sys.version) def get_host_settings(self): diff --git a/addon/flamenco/manager/docs/ShamanFileSpecWithStatus.md b/addon/flamenco/manager/docs/ShamanFileSpecWithStatus.md index 04cfb7c5..fad7b0b3 100644 --- a/addon/flamenco/manager/docs/ShamanFileSpecWithStatus.md +++ b/addon/flamenco/manager/docs/ShamanFileSpecWithStatus.md @@ -1,5 +1,6 @@ # ShamanFileSpecWithStatus +Specification of a file, which could be in the Shaman storage, or not, depending on its status. ## Properties Name | Type | Description | Notes diff --git a/addon/flamenco/manager/docs/ShamanFileSpecWithStatusAllOf.md b/addon/flamenco/manager/docs/ShamanFileSpecWithStatusAllOf.md deleted file mode 100644 index 22df2127..00000000 --- a/addon/flamenco/manager/docs/ShamanFileSpecWithStatusAllOf.md +++ /dev/null @@ -1,12 +0,0 @@ -# ShamanFileSpecWithStatusAllOf - - -## Properties -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**status** | [**ShamanFileStatus**](ShamanFileStatus.md) | | -**any string name** | **bool, date, datetime, dict, float, int, list, str, none_type** | any string name can be used but the value must be the correct type | [optional] - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/addon/flamenco/manager/model/shaman_file_spec_with_status.py b/addon/flamenco/manager/model/shaman_file_spec_with_status.py index 91cccdbc..1a97c573 100644 --- a/addon/flamenco/manager/model/shaman_file_spec_with_status.py +++ b/addon/flamenco/manager/model/shaman_file_spec_with_status.py @@ -30,15 +30,11 @@ from flamenco.manager.exceptions import ApiAttributeError def lazy_import(): - from flamenco.manager.model.shaman_file_spec import ShamanFileSpec - from flamenco.manager.model.shaman_file_spec_with_status_all_of import ShamanFileSpecWithStatusAllOf from flamenco.manager.model.shaman_file_status import ShamanFileStatus - globals()['ShamanFileSpec'] = ShamanFileSpec - globals()['ShamanFileSpecWithStatusAllOf'] = ShamanFileSpecWithStatusAllOf globals()['ShamanFileStatus'] = ShamanFileStatus -class ShamanFileSpecWithStatus(ModelComposed): +class ShamanFileSpecWithStatus(ModelNormal): """NOTE: This class is auto generated by OpenAPI Generator. Ref: https://openapi-generator.tech @@ -112,16 +108,20 @@ class ShamanFileSpecWithStatus(ModelComposed): read_only_vars = { } + _composed_schemas = {} + @classmethod @convert_js_args_to_python_args - def _from_openapi_data(cls, *args, **kwargs): # noqa: E501 + def _from_openapi_data(cls, sha, size, path, status, *args, **kwargs): # noqa: E501 """ShamanFileSpecWithStatus - a model defined in OpenAPI - Keyword Args: + Args: sha (str): SHA256 checksum of the file size (int): File size in bytes path (str): Location of the file in the checkout status (ShamanFileStatus): + + Keyword Args: _check_type (bool): if True, values for parameters in openapi_types will be type checked and a TypeError will be raised if the wrong type is input. @@ -179,29 +179,18 @@ class ShamanFileSpecWithStatus(ModelComposed): self._configuration = _configuration self._visited_composed_classes = _visited_composed_classes + (self.__class__,) - constant_args = { - '_check_type': _check_type, - '_path_to_item': _path_to_item, - '_spec_property_naming': _spec_property_naming, - '_configuration': _configuration, - '_visited_composed_classes': self._visited_composed_classes, - } - composed_info = validate_get_composed_info( - constant_args, kwargs, self) - self._composed_instances = composed_info[0] - self._var_name_to_model_instances = composed_info[1] - self._additional_properties_model_instances = composed_info[2] - discarded_args = composed_info[3] - + self.sha = sha + self.size = size + self.path = path + self.status = status for var_name, var_value in kwargs.items(): - if var_name in discarded_args and \ + if var_name not in self.attribute_map and \ self._configuration is not None and \ self._configuration.discard_unknown_keys and \ - self._additional_properties_model_instances: + self.additional_properties_type is None: # discard variable. continue setattr(self, var_name, var_value) - return self required_properties = set([ @@ -211,20 +200,19 @@ class ShamanFileSpecWithStatus(ModelComposed): '_path_to_item', '_configuration', '_visited_composed_classes', - '_composed_instances', - '_var_name_to_model_instances', - '_additional_properties_model_instances', ]) @convert_js_args_to_python_args - def __init__(self, *args, **kwargs): # noqa: E501 + def __init__(self, sha, size, path, status, *args, **kwargs): # noqa: E501 """ShamanFileSpecWithStatus - a model defined in OpenAPI - Keyword Args: + Args: sha (str): SHA256 checksum of the file size (int): File size in bytes path (str): Location of the file in the checkout status (ShamanFileStatus): + + Keyword Args: _check_type (bool): if True, values for parameters in openapi_types will be type checked and a TypeError will be raised if the wrong type is input. @@ -280,49 +268,18 @@ class ShamanFileSpecWithStatus(ModelComposed): self._configuration = _configuration self._visited_composed_classes = _visited_composed_classes + (self.__class__,) - constant_args = { - '_check_type': _check_type, - '_path_to_item': _path_to_item, - '_spec_property_naming': _spec_property_naming, - '_configuration': _configuration, - '_visited_composed_classes': self._visited_composed_classes, - } - composed_info = validate_get_composed_info( - constant_args, kwargs, self) - self._composed_instances = composed_info[0] - self._var_name_to_model_instances = composed_info[1] - self._additional_properties_model_instances = composed_info[2] - discarded_args = composed_info[3] - + self.sha = sha + self.size = size + self.path = path + self.status = status for var_name, var_value in kwargs.items(): - if var_name in discarded_args and \ + if var_name not in self.attribute_map and \ self._configuration is not None and \ self._configuration.discard_unknown_keys and \ - self._additional_properties_model_instances: + self.additional_properties_type is None: # discard variable. continue setattr(self, var_name, var_value) if var_name in self.read_only_vars: raise ApiAttributeError(f"`{var_name}` is a read-only attribute. Use `from_openapi_data` to instantiate " f"class with read only attributes.") - - @cached_property - def _composed_schemas(): - # we need this here to make our import statements work - # we must store _composed_schemas in here so the code is only run - # when we invoke this method. If we kept this at the class - # level we would get an error because the class level - # code would be run when this module is imported, and these composed - # classes don't exist yet because their module has not finished - # loading - lazy_import() - return { - 'anyOf': [ - ], - 'allOf': [ - ShamanFileSpec, - ShamanFileSpecWithStatusAllOf, - ], - 'oneOf': [ - ], - } diff --git a/addon/flamenco/manager/model/shaman_file_spec_with_status_all_of.py b/addon/flamenco/manager/model/shaman_file_spec_with_status_all_of.py deleted file mode 100644 index bbfbdb5a..00000000 --- a/addon/flamenco/manager/model/shaman_file_spec_with_status_all_of.py +++ /dev/null @@ -1,267 +0,0 @@ -""" - Flamenco manager - - Render Farm manager API # noqa: E501 - - The version of the OpenAPI document: 1.0.0 - Generated by: https://openapi-generator.tech -""" - - -import re # noqa: F401 -import sys # noqa: F401 - -from flamenco.manager.model_utils import ( # noqa: F401 - ApiTypeError, - ModelComposed, - ModelNormal, - ModelSimple, - cached_property, - change_keys_js_to_python, - convert_js_args_to_python_args, - date, - datetime, - file_type, - none_type, - validate_get_composed_info, - OpenApiModel -) -from flamenco.manager.exceptions import ApiAttributeError - - -def lazy_import(): - from flamenco.manager.model.shaman_file_status import ShamanFileStatus - globals()['ShamanFileStatus'] = ShamanFileStatus - - -class ShamanFileSpecWithStatusAllOf(ModelNormal): - """NOTE: This class is auto generated by OpenAPI Generator. - Ref: https://openapi-generator.tech - - Do not edit the class manually. - - Attributes: - allowed_values (dict): The key is the tuple path to the attribute - and the for var_name this is (var_name,). The value is a dict - with a capitalized key describing the allowed value and an allowed - value. These dicts store the allowed enum values. - attribute_map (dict): The key is attribute name - and the value is json key in definition. - discriminator_value_class_map (dict): A dict to go from the discriminator - variable value to the discriminator class name. - validations (dict): The key is the tuple path to the attribute - and the for var_name this is (var_name,). The value is a dict - that stores validations for max_length, min_length, max_items, - min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum, - inclusive_minimum, and regex. - additional_properties_type (tuple): A tuple of classes accepted - as additional properties values. - """ - - allowed_values = { - } - - validations = { - } - - @cached_property - def additional_properties_type(): - """ - This must be a method because a model may have properties that are - of type self, this must run after the class is loaded - """ - lazy_import() - return (bool, date, datetime, dict, float, int, list, str, none_type,) # noqa: E501 - - _nullable = False - - @cached_property - def openapi_types(): - """ - This must be a method because a model may have properties that are - of type self, this must run after the class is loaded - - Returns - openapi_types (dict): The key is attribute name - and the value is attribute type. - """ - lazy_import() - return { - 'status': (ShamanFileStatus,), # noqa: E501 - } - - @cached_property - def discriminator(): - return None - - - attribute_map = { - 'status': 'status', # noqa: E501 - } - - read_only_vars = { - } - - _composed_schemas = {} - - @classmethod - @convert_js_args_to_python_args - def _from_openapi_data(cls, status, *args, **kwargs): # noqa: E501 - """ShamanFileSpecWithStatusAllOf - a model defined in OpenAPI - - Args: - status (ShamanFileStatus): - - Keyword Args: - _check_type (bool): if True, values for parameters in openapi_types - will be type checked and a TypeError will be - raised if the wrong type is input. - Defaults to True - _path_to_item (tuple/list): This is a list of keys or values to - drill down to the model in received_data - when deserializing a response - _spec_property_naming (bool): True if the variable names in the input data - are serialized names, as specified in the OpenAPI document. - False if the variable names in the input data - are pythonic names, e.g. snake case (default) - _configuration (Configuration): the instance to use when - deserializing a file_type parameter. - If passed, type conversion is attempted - If omitted no type conversion is done. - _visited_composed_classes (tuple): This stores a tuple of - classes that we have traveled through so that - if we see that class again we will not use its - discriminator again. - When traveling through a discriminator, the - composed schema that is - is traveled through is added to this set. - For example if Animal has a discriminator - petType and we pass in "Dog", and the class Dog - allOf includes Animal, we move through Animal - once using the discriminator, and pick Dog. - Then in Dog, we will make an instance of the - Animal class but this time we won't travel - through its discriminator because we passed in - _visited_composed_classes = (Animal,) - """ - - _check_type = kwargs.pop('_check_type', True) - _spec_property_naming = kwargs.pop('_spec_property_naming', False) - _path_to_item = kwargs.pop('_path_to_item', ()) - _configuration = kwargs.pop('_configuration', None) - _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) - - self = super(OpenApiModel, cls).__new__(cls) - - if args: - raise ApiTypeError( - "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( - args, - self.__class__.__name__, - ), - path_to_item=_path_to_item, - valid_classes=(self.__class__,), - ) - - self._data_store = {} - self._check_type = _check_type - self._spec_property_naming = _spec_property_naming - self._path_to_item = _path_to_item - self._configuration = _configuration - self._visited_composed_classes = _visited_composed_classes + (self.__class__,) - - self.status = status - for var_name, var_value in kwargs.items(): - if var_name not in self.attribute_map and \ - self._configuration is not None and \ - self._configuration.discard_unknown_keys and \ - self.additional_properties_type is None: - # discard variable. - continue - setattr(self, var_name, var_value) - return self - - required_properties = set([ - '_data_store', - '_check_type', - '_spec_property_naming', - '_path_to_item', - '_configuration', - '_visited_composed_classes', - ]) - - @convert_js_args_to_python_args - def __init__(self, status, *args, **kwargs): # noqa: E501 - """ShamanFileSpecWithStatusAllOf - a model defined in OpenAPI - - Args: - status (ShamanFileStatus): - - Keyword Args: - _check_type (bool): if True, values for parameters in openapi_types - will be type checked and a TypeError will be - raised if the wrong type is input. - Defaults to True - _path_to_item (tuple/list): This is a list of keys or values to - drill down to the model in received_data - when deserializing a response - _spec_property_naming (bool): True if the variable names in the input data - are serialized names, as specified in the OpenAPI document. - False if the variable names in the input data - are pythonic names, e.g. snake case (default) - _configuration (Configuration): the instance to use when - deserializing a file_type parameter. - If passed, type conversion is attempted - If omitted no type conversion is done. - _visited_composed_classes (tuple): This stores a tuple of - classes that we have traveled through so that - if we see that class again we will not use its - discriminator again. - When traveling through a discriminator, the - composed schema that is - is traveled through is added to this set. - For example if Animal has a discriminator - petType and we pass in "Dog", and the class Dog - allOf includes Animal, we move through Animal - once using the discriminator, and pick Dog. - Then in Dog, we will make an instance of the - Animal class but this time we won't travel - through its discriminator because we passed in - _visited_composed_classes = (Animal,) - """ - - _check_type = kwargs.pop('_check_type', True) - _spec_property_naming = kwargs.pop('_spec_property_naming', False) - _path_to_item = kwargs.pop('_path_to_item', ()) - _configuration = kwargs.pop('_configuration', None) - _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) - - if args: - raise ApiTypeError( - "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( - args, - self.__class__.__name__, - ), - path_to_item=_path_to_item, - valid_classes=(self.__class__,), - ) - - self._data_store = {} - self._check_type = _check_type - self._spec_property_naming = _spec_property_naming - self._path_to_item = _path_to_item - self._configuration = _configuration - self._visited_composed_classes = _visited_composed_classes + (self.__class__,) - - self.status = status - for var_name, var_value in kwargs.items(): - if var_name not in self.attribute_map and \ - self._configuration is not None and \ - self._configuration.discard_unknown_keys and \ - self.additional_properties_type is None: - # discard variable. - continue - setattr(self, var_name, var_value) - if var_name in self.read_only_vars: - raise ApiAttributeError(f"`{var_name}` is a read-only attribute. Use `from_openapi_data` to instantiate " - f"class with read only attributes.") diff --git a/addon/flamenco/manager/models/__init__.py b/addon/flamenco/manager/models/__init__.py index 983bc1d0..e97d5117 100644 --- a/addon/flamenco/manager/models/__init__.py +++ b/addon/flamenco/manager/models/__init__.py @@ -29,7 +29,6 @@ from flamenco.manager.model.security_error import SecurityError from flamenco.manager.model.shaman_checkout import ShamanCheckout from flamenco.manager.model.shaman_file_spec import ShamanFileSpec from flamenco.manager.model.shaman_file_spec_with_status import ShamanFileSpecWithStatus -from flamenco.manager.model.shaman_file_spec_with_status_all_of import ShamanFileSpecWithStatusAllOf from flamenco.manager.model.shaman_file_status import ShamanFileStatus from flamenco.manager.model.shaman_requirements_request import ShamanRequirementsRequest from flamenco.manager.model.shaman_requirements_response import ShamanRequirementsResponse diff --git a/addon/flamenco/manager_README.md b/addon/flamenco/manager_README.md index c3093e59..0df9b5ca 100644 --- a/addon/flamenco/manager_README.md +++ b/addon/flamenco/manager_README.md @@ -4,7 +4,7 @@ Render Farm manager API The `flamenco.manager` package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project: - API version: 1.0.0 -- Package version: 8a43c69f-dirty +- Package version: 8a97cf50-dirty - Build package: org.openapitools.codegen.languages.PythonClientCodegen For more information, please visit [https://flamenco.io/](https://flamenco.io/) @@ -104,7 +104,6 @@ Class | Method | HTTP request | Description - [ShamanCheckout](flamenco/manager/docs/ShamanCheckout.md) - [ShamanFileSpec](flamenco/manager/docs/ShamanFileSpec.md) - [ShamanFileSpecWithStatus](flamenco/manager/docs/ShamanFileSpecWithStatus.md) - - [ShamanFileSpecWithStatusAllOf](flamenco/manager/docs/ShamanFileSpecWithStatusAllOf.md) - [ShamanFileStatus](flamenco/manager/docs/ShamanFileStatus.md) - [ShamanRequirementsRequest](flamenco/manager/docs/ShamanRequirementsRequest.md) - [ShamanRequirementsResponse](flamenco/manager/docs/ShamanRequirementsResponse.md) diff --git a/addon/flamenco/operators.py b/addon/flamenco/operators.py index ffc641a3..b6b21b24 100644 --- a/addon/flamenco/operators.py +++ b/addon/flamenco/operators.py @@ -301,11 +301,9 @@ class FLAMENCO_OT_submit_job(FlamencoOpMixin, bpy.types.Operator): target="/", # Target directory irrelevant for Shaman transfers. exclusion_filter="", # TODO: get from GUI. relative_only=True, # TODO: get from GUI. - packer_class=functools.partial( - bat_shaman.Packer, - api_client=self.get_api_client(context), - checkout_path=checkout_root, - ), + api_client=self.get_api_client(context), + checkout_path=checkout_root, + packer_class=bat_shaman.Packer, ) return checkout_root / blendfile.name # TODO: get relative to the checkout dir. diff --git a/pkg/api/flamenco-manager.yaml b/pkg/api/flamenco-manager.yaml index e9e82fad..a8d42bca 100644 --- a/pkg/api/flamenco-manager.yaml +++ b/pkg/api/flamenco-manager.yaml @@ -747,11 +747,29 @@ components: required: [sha, size, path] ShamanFileSpecWithStatus: - allOf: - - $ref: '#/components/schemas/ShamanFileSpec' - - properties: - "status": {$ref: "#/components/schemas/ShamanFileStatus"} - required: [status] + # Using allOf here would trigger a bug in the Python code generator, + # resulting in this error: + # + # Values stored for property status in ShamanFileSpecWithStatus differ + # when looking at self and self's composed instances. All values must be + # the same at ['['received_data', 'files', 0]']['status'] + # + # The underlying cause is that composited types can apparently store + # property values multiple times, and these have to be equal. However, one + # is using the `ShamanFileStatus` type, and the other is using the `str` + # type to store the same status value. + # + # To work around this, even though this should be a composition of + # `ShamanFileSpec` with some extra properties, we just repeat those same + # properties here. + type: object + description: Specification of a file, which could be in the Shaman storage, or not, depending on its status. + properties: + "sha": {type: string, description: "SHA256 checksum of the file"} + "size": {type: integer, description: "File size in bytes"} + "path": {type: string, description: "Location of the file in the checkout"} + "status": {$ref: "#/components/schemas/ShamanFileStatus"} + required: [sha, size, path, status] ShamanCheckout: type: object diff --git a/pkg/api/openapi_spec.gen.go b/pkg/api/openapi_spec.gen.go index 93d67260..7ea2f44c 100644 --- a/pkg/api/openapi_spec.gen.go +++ b/pkg/api/openapi_spec.gen.go @@ -18,93 +18,94 @@ import ( // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+Q8224cN5a/QtQskATbN11s2XpajR0nMpJYiOTJArEhsapOdVNikRWSpXbHEDAfsX+y", - "O8A+7DztD3j+aEEeVhWriy21E8nj2fGD0epiHR6e+439PslkWUkBwujk8H2iswWU1H080prNBeRnVF/Z", - "v3PQmWKVYVIkh72nhGlCibGfqCbM2L8VZMCuISfpipgFkJ+kugI1SUZJpWQFyjBwu2SyLKnI3WdmoHQf", - "/kVBkRwmf5h2yE09ZtNn+EJyM0rMqoLkMKFK0ZX9+1Km9m3/tTaKibn//rxSTCpmVsECJgzMQTUr8NvI", - "64KW8Qe3w9SGmvrO41j6neJKeyKqrzYjUtcstw8KqUpqkkP8YrS+8GaUKPilZgry5PDnZpEljj9Li1tw", - "hDUqBSQJsRp1/Hrb7ivTS8iMRfDomjJOUw4vZXoKxlh0BpJzysScA9H4nMiCUPJSpsRC0xEBWUiW4cc+", - "nJ8WIMicXYMYEc5KZpycXVPOcvt/DZoYab/TQDyQCXkl+IrU2uJIlswsCBLNbW73bkVwQPx1YcuhoDU3", - "Q7zOFkD8Q8SD6IVcCo8MqTUosrS452BAlUy4/RdMNySZIPgAZnyL9pupkZIbVvmNmOg2svKoCpqBAwo5", - "M/boCNHjX1CuYTQkrlmAskhTzuWS2FfXESW0MHbNAsilTMmCapICCKLrtGTGQD4hP8ma54SVFV+RHDjg", - "a5wTeMc0AqT6SpNCKgR9KdMRoSK3BkSWFeN2DTOTN6IT9FRKDlS4E11TPqTPycospCDwrlKgNZOO+CkQ", - "u7qmBnJLI6lyPGDDB3An6bOuxavlzWgoGlewGuJwnIMwrGCgPJBW5EekrLWx+NSC/VKjIHqmXXpFiO5j", - "FYOqeUQXjsSKwDujKKFqXpfWwjTyllariX1RT05lCSeoW6svvyKZZUOtIbcrMwXUAB7V698qwKFT8c6y", - "fIQIsbKEnFEDfEUUWFCEuqPmUDDB7Asjawjc9nbLkaOJrI3HiCrDsppT1fJhgzzoOm3M521WN2KoTv2b", - "rap/NIQz//o102xdyYyqbyOQVdy+anl5eH2MBtISq1ErRb7k7AoIJX/kIKwQ0zwfS/HVhJyCseAuHEMu", - "0MygP6YCbYGgvN3DLKixW9c8F184gWwtFYjcGRAdJ/Sai7EK4Bdt6RZOOz6teYc6HdsnKA6oEA3PybNa", - "KRCGr4i0dpw2cJ2GBZZcT8jFt0en3379/PzF8Xdfn58cnX17gVFKzhRkRqoVqahZkH8lF2+S6R/cvzfJ", - "BaFVZUma47FB1KU9X8E4nNv1ySjJmWo+uq+9R11QvYD8vFv5NqLAm4RmaOA9BYLTB1YD3RfV5Ph5o8/u", - "2FZovEhMyA+SCNDW1mmj6szUCjT50rkvPSI5y+xWVDHQXxGqgOi6qqQy60f3yI9sZLO3aw/NJTXJyMnC", - "nYeMn67x9t2eGCUyTb6ngs5BoQtgxqk+La2BjoQGnKbAPy5k88TcPtyMhTSDaGBNHbxIIHrBnnfphqVW", - "xLh/x7RphMFJ92a6DWnUhHG/7cRnPYu44bjdFrEDNvH64Fj+AVFgvbRzWZRoDA59lOks0TvIagN35RGb", - "g/RWgILHDXpxxgWvxE70tVJSWWDrmUwOvei80ZhhalCC1nQew3cNIQezWx/D5gWnJYhM/gmU9sHilpS5", - "7t64HYtmoderGBYvMfWinL8qksOfb5ew0yY+tG/djAaEdLFITGLsAxfNsRK0oWVl7VFD7pwaGNsnsdCJ", - "RcC9fn38vHEzL112dEditW1OZ01Fm9LVVX7Pp1njjsO0oVm3X4vs25u3yKDvwdCcGuoYlecu7KL8pEf7", - "wYnX4kyVMqOoWpHSA/NuV0/I91I5xa04vAt9TkaF9VqltPG/s1i11XJyQSfpJLsgQhqkQxMmX4ELPeEd", - "tbC8QDtBO0xOK8UMkBeKzRfWC9kYZQIlZdxivUoViH9LvQuUat6sQB1ITt0Ccmr+93+ugQeGrSfIp4GP", - "iNMJo7nou62ANA6UZoZdu8yZisxSAJPoioPxnwUSi0kxLijDFe2HitoQPRklv9RQuw9UZQt2HXxE/4zg", - "x1YynNv3QHpfuM8IpbYkGoebJ6NkSV2SNy6kGttIRkcdvPc1z6Qo2LxW1ETNjl7QkoqvhfUkeTR7x+B3", - "AeTULSV2R2IUFboARY5Ojl3E1nijyd1haH/LmJn6EeZMG1CQox8Zok3z3OaMUV3gVJtzx89+0SeIO1h2", - "tdkTcWqsfscDE1mYJVUbopatzA4eqbM8bZRw3hZw+lHAnTWO31VwamkxaokaFp4aYoySDKN6h2WyTuWA", - "MhtOFOPzKWS1Yma1wVVv7X9vc7wotc8WkF3JOlIHsrmYLJxUa6w1mQUwRU6/Pdp99Jhk9kVdlyOi2a8u", - "dE9XBjRGvjloiwLhMkPb6NPBzO/WpTFrlhK11jpgl4QcJl2GPZlLS8IFTQ6TvUfpbP/pTrZ7kM729vby", - "nSLdf1Rks4MnT+nObkZnj9Od/PH+LN999PjpwZNZ+mR2kMOj2X5+MNt9CjMLiP0KyeHO/u6+8+C4G5fz", - "uc3Ugq0e76UHu9njvfTp/u5+ke/spU/3DmZF+ng2e/x09mSW7dGdRwc7B1mxR/P9/d3He4/SnScH2WP6", - "5Omj2cHTbqvdA+fJ1suDSJETh8CgEGRzvOUCFNZ2vOXyOW+v6NHAGZFjX7/m1BrupozibVTLAJc9Uk0y", - "bwUhJ1KEm0zIsSCS2/Tcx0+6CTY8LLfvkmpyaRM7++BNexxy/PxNMiJpbZD1TDdQbD7vXSpFLFwx4ML7", - "yLHm9XyqMxAwtto3xRrT+Pj5RS+V75Tei8yW2QHi/oJxOK0guzM3QOCjPps2a1MLd6hNFWSsYF4dXE3E", - "+QuvGJ6k2khF5zBMD6qoeHwnO3gWSgixwTgaBi5oBMO+XocwozCcUK8DsQToG4RIAjF0e42SjJJqOwL/", - "xMyii1W2DNvXWT9w+Vu5qQCMd1Xr5wnj1sHqILSqxZWQS+GCXi5pjoGQlYGe++9ojsB+xL1clfRHjI5+", - "s/V21rrHr40G+YEs7yexsp/ARmyW2T6/dCWFhnhPBLlVKFkSSlTwGvFR8ChkJcpZY7YbCwLq2hrvFw6U", - "q5BSBcQJmvUFfpn9Dt5lvM4hxw0tDOWx+5Qy0EWHrT48jFiEG7Xqds+yEpil3ys12GbsG441Fff8/1hf", - "cs9mLnaCsEASLZ92YV3XbbPi2VSD1iSwDFL/h0um/YO9D/9B/vbnD3/58NcP//XhL3/784f//vDXD/8Z", - "NpsPH836pVi/y3lW5slh8t7/eeMCh1pcnaMQ7tkzGUUzc07rnMkmEbfM8wHoVLk3p7qYXspUYyC0s7s3", - "cSDDAsvJD9/YPyudHFolKhQtLXuTnfGOVTBW0jnoc6nOr1kO0rpf900ySmRtqtpgqR/eGRBYRUsmlfM/", - "iME5ruqjhJu0SAUyrpll1dgffIyvJAPdCvl4RwWqrfZsO8HQ9qoscyLjDAG77ip+NUuDXtrteZbPE/2M", - "QYtVTDeCgYmPqLK09ZS2AGLTyq7eEqme+MpLLI6wOLx2dbaIH2qfEdfOE4akK0J94drqKFbosCOMJuhN", - "PZvtPiZczr05crM0zHyhffnbd57XShVBJaKPwysBY86Eb76K3IbNQJYLaiFmbRNt4bpdTMxbr+g2npBX", - "16CW1jZoUim4ZrLWfIVnaTZt636x2JbLeSzYnhOLVNDst7tZl8y5S2d8780i7UjhNgSqOMOK/7Be0ZOF", - "bcdsYmU/5A6WhzZVtH5HcQcyBSb+6HcWadadCu7Uq69EtwjqM2830uOUzcWrj6VEU68539xfuPdjB7Wm", - "DacdYHXLqQ018GxBxRwiZU0nROedofioolw0BAiAbYVUvgmre8DlDgz6RlcbqgwmXXRJr1ylT3OAygYf", - "rvJmU+Xa5JikGdB+tSwKawkithWVxdXuTi3WeLylQ+Cc1rEk/rUGZXlvza01YbiYHD8fkYpqvZQqbx6h", - "duDQGKGmWaoCtbd2xtHLFZqpZllneBbGVMmNxZGJQmLPTxiama7N1rbjyBlQq3y14v5NfTidFk14xuR0", - "2F35Eac5XlBVktKXqo5OjpNRwlkGPuvx+3xz8t313gD+crmczEVto7Wpf0dP5xUf701mExCThSmx7cEM", - "72Hrt0uCrmCyM5lNZna1rEDQitnQzn2FVQbHmSmt2DRb7wTM0dhZCXXfHduo7hsw/ZaBlT/Mlxyo3dms", - "ISkI9z6tKu4rPtNLjaBRlu+S9GiLwnGuT3Fho0He5m0of3VZUrVCjLEUFIJp55SClruhNi762YVnydse", - "jK9FXkkmjHN6cz94MwDY8qEFejNC2too1um71BGaYqKAzUtvRf4o89W90bHfqR3Sz01ySJ+CJKFBsZH5", - "zQNy+BaEllQTXWcZ6KLmfEVwjtAN/flw6JrlNeU4ejhZG+a8F+yw7xDBzz0gTVuhL25IbEKJgKUb9rAi", - "sy4ZwUhEKHlOTPqS97IZGcMJSPCC2Betaet5N+lsO5fygMwcDsFESNcu6gZhIvrKB8Mybo7EFXz6s0S3", - "kK7bqiX/ZTeh3KPf+0uZnrP8ZiMJX4DJFqih4SjKz+8TZk/lR8m8VUdgA0UaBXS8qxf39u+jdM4j9tnh", - "Tu4eEJriLKfj3RZyiy+J3Pul0mLekD0IKzfJ7J/agZUHI8X62M1vdi6thDX9nTX/crt7ecaZq29mVJBa", - "+16WkVifxr+YtklbTa0ppN12vtDZkhVjoanyHfLxsmuQR11P00r3jfSH8T+RtCxC6C61brD/pK5oMFSw", - "jSx8Qp9TC3hXQWYgJ+DXhCLUoO8dz7LhZyN1/ou3kZe6AKZ7U69LlGZzMZZFcUsUY9PMohiq6/4w2v/8", - "COnTFWfSe4nKz2+tMe5o9j1VV2GGQjVpEqE7qP2Mcj86hRLmVJx7A9IEBlfCzZDD6gsFZC7xbo0DP4mz", - "RNzBEfGgSu232KzOba3zU+rysALwD6HMW8vgUW0WIAwWBH3Z0UpD0/tYtuO19yyQCmi+sqssPBxy6ZVC", - "WcfwobgaX2mN+vuAZcnfWzIcpiRzz0lX1rkZbTJmZPMbn7dIfbx4YEiy7Ob+FODFlNUGIsTlYJwFRbCo", - "8YoUzB7UkIUbRcj7Q+sa8Zxb2LP/X37P23PPNyTChJy5qSs3h5W6yyw0swaDQ47xPjZCvC3pGjM9WRkR", - "qazlaqjS2BdQYy4zyp1po1zftz27ht5paj0QVeMvTW9wr9kC8prDGU48PlxeHV7hjjDWXd4OCwqbDNUP", - "0t/T7F+5cvlFcyPjZpTsz/bur/TUG+GMIH8CqqltPAfB0Gjuz55GZo1RAJkmQprG02HHEMVpRLRsHrvr", - "rtC7eoJHd11yIuQSj7q792ldS6NFVFgsZWooEy7sdtjhqKC7ITaX7taukM7OorZ9pMa+Qui0hR9Q4y5V", - "cjKlvYCrSNkp0JDpe9ej8eWTuK4EvdZtKige4O8vody/uwhOskkXfTzEBKLY1DA+2lucLaCBtXSmNYOq", - "8ahRFTnzvV/nkb3VCMUImeb0xPRhO50J4f+juKXXXRse+9BmVbHMlUnCrnml5FyB1iN/t8VfVlakoIzX", - "Cu70LY1H0SDyXjXMkruBbq2YjYhQTfAiw7QZQZ3ioPIt/qQ/iv5AvYD+JhGG9GYm24hP1mby26S42cvJ", - "mb/hFFb2Q3F+WFFrMaEcExn38wXae4L9h0fgzIXLS/sfkte5PjGfkNcayIXu8SYcmLywjMAJdeJIuWDZ", - "gkgBevI5FaGe4Sh+cD8bc0S9KjkTV366EiXIUwC7OcZGki1RrP+jnJMFvQb8LQocVURj5gf7UijcVTXK", - "efuLFp2b6rQZibqmzaceIUp0KO0Omd4NDaqAxrU5HEzdVqdDlj6ofseGo7dV9U9aqbllNjiGb516flkm", - "WYpD3psQHjUWH0UCiB+mxSN+XrriZs8JbeQ5pIFDt/kBlUoqo73GI6eoag92p6Qf2UDYbpO5HkOYwvcB", - "djmBH6XG1gJi0dkb/JUBwzjvUAjUw8Gbvm8G62+m79037Fe4QeWwxNGb9ATnbKWCZ14Q1yLFra9quB8B", - "GoaVzdJb48rBLMvwp49+hfW7Ju2lgciuDQW22bW7HfL2wbVuMFsdK3v0CxafnwaFA5jdDHj0NgBenBoq", - "y22Wu5XIf25hHMUyDW9Rmhjb37PwF9tyKECR9ooB+mdHDefp3yS7sydvkq7m48ZHXU4s+IqkNk4wtbL5", - "i/v5m+54uo3ecC6kvdMxYDhm05RriTC0LEEKIMC1g9ON0MbQdNLiCLgAmrtemifhv49xm/EzKsbP7TnH", - "rx2AJELD4JZzjIZSsTkTlLs9LfwJOS78jC6X4Uxve/eFmXbWlgl/d4WFJtuN3Y68zba8oMytyCGt8WLn", - "Fmd75REbv/CIJbeJ5da5tswMmLE2CmjZtxBtOp8yYfV7mNAP43ncQ69d0vuNmbYTr0GevTt7ctdyL449", - "QQz68vs7B1EIyr9uk4CSmmxBUjBL8MLuyRlMuzQjMH4OABFwl9akGtidNmBuZNmlOI8i2Rkqsb9Le4fW", - "NhrYaY4XvErJDLRjRAr2xXb/dNXTOwwnLjaq0CGxPLvA6S+0LiE5/Ek+Fw/kPIMvsG32O+QH6SoU1Awf", - "Ov0spMpYylck41JjLePbs7MTkkkhwP2qAxqwpozjDW/BBNML0D1+AYF3NDNE0xJ8GGmkm/+3r+SythEe", - "vqAnb0TD1S/cHWjUJi8LKcQ4QFKZrza60rAuY7foUoshWXyhx35Gh4pDsNMkaEwNfo6vP4Y0GKtjRgMv", - "Jp09c8M2Q9P7UqZN39QVcH6pQTHQo2DUbrQ2uTTpzXfpCNCjk+P+sF/YNpNlWQt/g8Oa9OGsaAve158i", - "vh7pd3RyPHIbOZHrmO8P5Eos9u9LmbaJrA7ge37dvL35vwAAAP//hZ9HEDhWAAA=", + "H4sIAAAAAAAC/+Q87W4cN5KvQvQekF3cfOnDlq1f57XjREESC5G8OSA2JHZ3zQwtNtkh2RpPDAH7EPcm", + "dwvcj9tf9wLeNzoUi/01zZHGieX17vmHMZpmVxXru4rFeZdkuii1AuVscvwusdkSCu4/PrFWLBTk59xe", + "4d852MyI0gmtkuPeUyYs48zhJ26ZcPi3gQzENeQsXTO3BPajNldgJskoKY0uwTgBHkumi4Kr3H8WDgr/", + "4V8MzJPj5HfTlrhpoGz6lF5IbkaJW5eQHCfcGL7Gv9/oFN8OX1tnhFqE7y9KI7QRbt1ZIJSDBZh6BX0b", + "eV3xIv7gdpjWcVfduR3k3xmtxB1xe7WdkKoSOT6Ya1NwlxzTF6PNhTejxMDPlTCQJ8c/1YuQOWEvDW2d", + "LWxwqcOSLlWjVl6vG7w6fQOZQwKfXHMheSrhG52egXNIzkBzzoRaSGCWnjM9Z5x9o1OG0GxEQZZaZPSx", + "D+fHJSi2ENegRkyKQjivZ9dcihz/r8Ayp/E7CywAmbAXSq5ZZZFGthJuyYhpHjniblRwwPxNZcthzivp", + "hnSdL4GFh0QHs0u9UoEYVlkwbIW05+DAFEJ5/Etha5ZMCHwHZhxF883UaS2dKAMioVpEqI9mzjPwQCEX", + "DrdOEAP9cy4tjIbMdUswSDSXUq8YvrpJKONzh2uWwN7olC25ZSmAYrZKC+Ec5BP2o65kzkRRyjXLQQK9", + "JiWDt8ISQG6vLJtrQ6Df6HTEuMrRgeiiFBLXCDd5pVpFT7WWwJXf0TWXQ/6crt1SKwZvSwPWCu2ZnwLD", + "1RV3kCOPtMlpg7UcwO+kL7qGrkY2o6FqXMF6SMNJDsqJuQATgDQqP2JFZR3SUynxc0WKGIT2JhhCFA8a", + "BjeLiC08UWsGb53hjJtFVaCHqfUtLdcTfNFOznQBp2Rb69//gWUohspCjiszA9wBbTXY37pDQ2virWf5", + "ABUSRQG54A7kmhlAUIz7reYwF0rgCyN0BB49ohx5nujKBYq4cSKrJDeNHLbog63S2n3e5nUjjuosvNmY", + "+gdDOA+vXwsrNo3Mmeo2BqHh9k0r6MPLE3KQyKzarAz7vRRXwDj7owSFSszzfKzVHybsDByCu/QCuSQ3", + "Q/GYK/IFissGh1tyh6grmasvvEI2ngpU7h2IjTN6I8SgAYRFO4aFs1ZOG9GhSsf4hNSBDKKWOXtaGQPK", + "yTXT6Md5DddbWMeT2wm7/PrJ2ddfPrt4fvLtlxenT86/vqQsJRcGMqfNmpXcLdm/sstXyfR3/t+r5JLx", + "skSW5rRtUFWB+5sLCRe4PhkluTD1R/91iKhLbpeQX7QrX0cMeJvSDB184EBn9x2vQeGLW3byrLZnv21U", + "mqASE/a9Zgos+jrrTJW5yoBlv/fhy45YLjJExY0A+wfGDTBblaU2bnPrgfgRZjYH+7hpqblLRl4X7txk", + "fHd1tG9xUpYoLPuOK74AQyFAOG/6vEAHHUkNJE9BfljKFpi5e7oZS2kG2cCGOQSVIPI6OO+yDeRWxLl/", + "K6yrlcFr93a+DXlUp3G/bsfnPY+4ZbstitgG63x9sK3wgBnAKO1DFmeWksOQZXpP9BayysFddcT2JL1R", + "oM7jmry44DqvxHb0pTHaILDNSiaHXnZeW8ywNCjAWr6I0btBkIfZro9R81zyAlSm/wTGhmRxR85ct2/c", + "TkW9MNhVjIpvqPTiUr6YJ8c/3a5hZ3V+iG/djAaM9LlITGPwgc/mRAHW8aJEf1SzO+cOxvgkljqJCLiX", + "L0+e1WHmG18d3VFY7VrToatoSrqqzD/ybjak4ymtedbia4h9ffOaBPQdOJ5zx72g8tynXVye9ng/2PFG", + "nmlS4Qw3a1YEYCHs2gn7ThtvuKWEt92Yk3GFUavQmP97j1WhlbNLPkkn2SVT2hEf6jT5CnzqCW85wgoK", + "7RXtODkrjXDAnhuxWGIUwhxlAgUXEqlepwbUv6UhBGqzqFeQDSRnfgE7c//7P9cgO46tp8hnnRgR5xNl", + "c9F3GwWpAyjPnLj2lTNXGXKAiuhSggufFTFLaDWec0Ermg8lxxQ9GSU/V1D5D9xkS3Hd+UjxmcCPUTN8", + "2A9Ael/4zwSlQhaNu8iTUbLivsgbz7UZYyZjowE+xJqnWs3FojLcRd2OXfKCqy8VRpI8Wr1T8rsEduaX", + "MsTInOHKzsGwJ6cnPmOro9Hk7jS0jzLmpn6AhbAODOQUR4Zk8zzHmjFqC5Jbd+Hl2W/6dPIOkV1tj0SS", + "O7TveGKi527FzZasZSe3Q1tqPU+TJVw0DZx+FnBnj+M3NZwaXowapnYbTzUzRklGWb2nMtnkcoczW3YU", + "k/MZZJURbr0lVO8cf28LvKS1T5eQXekq0gfCWkzPvVZb6jW5JQjDzr5+sv/gIcvwRVsVI2bFLz51T9cO", + "LGW+OVgkgUmdkW8M5WAWsLVlzIanJKvFAOyLkOOkrbAnC40sXPLkODl4kM4OH+9l+0fp7ODgIN+bp4cP", + "5tns6NFjvref8dnDdC9/eDjL9x88fHz0aJY+mh3l8GB2mB/N9h/DDAGJXyA53jvcP/QRnLBJvVhgpdZB", + "9fAgPdrPHh6kjw/3D+f53kH6+OBoNk8fzmYPH88ezbIDvvfgaO8omx/w/PBw/+HBg3Tv0VH2kD96/GB2", + "9LhFtX/kI9lme5A4cuoJGDSCsMZbLcFQbyd4rlDz9poeNZwROwn9a8nRcddtlOCjGgH46pFblgUvCDnT", + "qotkwk4U0xLL85A/2TrZCLA83hW37A0WdvjgVbMddvLsVTJiaeVI9MLWULCeDyGVExW+GXAZYuTYymox", + "tRkoGKP1TanHND55dtkr5VujDyqzY3VAtD8XEs5KyO6sDQj4qC+m7dbUwB1aUwmZmItgDr4n4uNFMIzA", + "Uuu04QsYlgdlVD2+1S08hNKFWFMcTQOXPEJh3667MKMwvFJvAkEG9B1CpIAYhr3aSEZJuRuDfxRu2eYq", + "O7F6xFZLkS1Z5m0n3cL6EdMGU7oRy6EElfsWv/KlPPn+f3LZ7BqsO+IIAftOqbYZ/e3iHaSglbpSeqV8", + "cSA1zylhRIH10qR2/wTsB6LGd5N/oCzyV0c5H9V6vNsauO4pQn2SaPQJfOl24fflZUutLMTPjkhac6ML", + "xpnpvMZCtTDqipK0rg5vtbmDucYg99yD8p1kboB5RcOYGZbhd/A2k1UOOSFEGCZQ9yl1oDXMxh7uRy26", + "iBpz+8i60nHfv1Vr6Di27zg2TDzI/0Nj7sdyhLc4vW4jKdpmbtPf9lQS1bPumm1oYNFpkdxf0yE8OHj/", + "H+xvf37/l/d/ff9f7//ytz+//+/3f33/n91D+eMHs37LOmC5yIo8OU7ehT9vfIJVqasLUsID3JMzPHMX", + "vMqFrhsWKLyQqE+Nf3Nq59M3OrWUMO7tH0w8yG4j6vT7r/DP0ibHaERzwwsUb7I33kMDEwVfgL3Q5uJa", + "5KAxFPpvklGiK1dWjo5E4K0DRd3GZFL6+EMUXNCqPkmEpCGqo+NWoKjGYeNjeiUZ2FZXjnd06pqu2K6T", + "Hs2ZHgonEv474rqrSVgv7Zw53l6Phno6zGI0VMVsozNY8gHdqKbv1DSKsPxu+1KRLlPoUMXyCKThpe9H", + "RuJQ84z5Y0/lWLpmPDT40Uapk0kn5+SCXlWz2f5DJvUiuCM/cyTcFzYcE4QT+o2WTqdj06fhhYKxFCoc", + "Uqscc15gqyVHiFlz2Lj0p4KYydZR0SOesBfXYFboGywrDVwLXVm5pr3USJv+aCzPlHoRS3wXDInqDEUg", + "NgzJUvqyL5xRItGeFR4hcCMFnYwM+zo9Xdh1HCnWHiXpUBttW+fvNzTBIDPg4o9+YzNrM6gQpl4fKoqi", + "08d6vZUfZ2KhXnwoJ+q+1sX2c5iPvu1OT27LbgdU3bJrxx08XXK1gEj71yvRResoPqh5GU0BOsB2Iirf", + "RtVHoOUOCvpO1zpuHBVdfMWvfEfUSoASkw/focSytXI5FWkObFit53P0BBHfSsbie5xnSDVtb+UJuOBV", + "rKB+acGg7NHdogujxezk2YiV3NqVNnn9iKyDhusYd/VS0zF79DOeX74hz63IWsezdK5MbpBGoeaazkaV", + "45lrjyObY0t2DhyNrzIyvGmPp9N5nZ4JPR2eQv1AUy/PuSlYEVp6T05PklEiRQah6gl4vjr99vpgAH+1", + "Wk0WqsJsbRresdNFKccHk9kE1GTpCjoeEk72qA3oks7pabI3mU1muFqXoHgpMLXzX1Hd7iUz5aWYZpsn", + "Jgtydqih/rsTzOq+Atc/WkH9o3rJg9qfzWqWgvLv87KUoV0zfWMJNOnyXZoePcrxkutzXGE2KJu6jfSv", + "Kgpu1kQxtWW6YJp5rs5oguOYF/3k07PkdQ/GlyovtVDOB71FGFAaAGzk0AC9GRFvMYv19q5thKdUKNAh", + "b/Aif9T5+qPxsX+iPeSfn3jRoQRJug4FM/Obe5TwLQStuGW2yjKw80rKNaN5Sz8cGdKha5FXXNKI5mRj", + "6PWjUEfnMxH6/ANWH7/01Y2YzThTsPJDMagym5rRGR3pap5Xk77mfVOP1tGkKARF7KvWtIm822y2md+5", + "R2EOh4UirGsWtQNDEXuVg6EiP2/jGz79matbWNeiatj/pp3k7vHv3RudXoj8ZisLn4PLlmSh3ZGdn94l", + "AncVOqDBqxOwgSGNOny868zy9d/H6HxE7IvD79w/YDylmVcvux30ll5SeYhLBVJes72TVm7T2T81gz33", + "xorN8aRfHVwaDavPwTbiy+3h5akUvr+ZccUqG878nKb+NP0lLBZtFUdXyFt0odHZsJVyoakJkwTjVTtI", + "EA099chBGDi4n/gTKcsijG5L65r6TxqKBsMXu+jCJ4w5lYK3JWQOcgZhTVeFavJD4FnV8qy1LnzxOvJS", + "m8C0b9pNjbJiocZ6Pr8li8Eycz4fmuvhMNv//BgZyhXv0nuFyk+v0Rm3PPuOm6tuhcItqwuhO7j9lMsw", + "YkYa5k1cBgdSJwZXys/aw/oLA2yh6Q6SBz+Ji0TdIRF1r0YdUGw356bX+SltedgB+Icw5p118EnllqAc", + "NQRD2xG1oT77WDVjyB9ZIQ3wfI2rEB4NA/VaoaIV+FBdXei0RuN9R2TJ31szPKUs889Z29a5GW1zZmz7", + "G5+3Sn24elBKsmrnIw3QBZ71FibE9WCcdZpgUecVaZjdqyPrIoqw9/smNNI+d/Bn/1xxL/jzIDdiwoSd", + "++m0euaGY5KKDkNCTvk+HYQEX9IezPR0xQ/lCNVwpfYvYMZSZ1x618al/dj+7Bp6u6nsQFVduFy+Jbxm", + "S8grCec0GXp/dXX3qntEsP6Se7ehsM1Rfa/Dfdb+1TRfX9Q3V25GyeHs4OO1nnqjrhHiT8HUvY1noAQ5", + "zcPZ48hMNimgsExpV0c6OjEkdRoxq+vH/low9K7o0Nb9KTlTekVb3T/4tKGltiKukEqdOi6UT7s9dTRS", + "6W/SLbS/3ay097NkbR9osS8IOm/gd7hxlyl5nbJBwU2k7dSxkOk7f0YT2idxW+mcte7SQQkAf3sL5eOH", + "i85OttliyIeEIhLrHsYHR4vzJdSwVt61ZlDWETVqIufh7NdH5OA1umpEQvN24vqwvc104f+jhKWX7TE8", + "nUO7dSky3ybpnpqXRi8MWDsKd4DCpW7D5lzIysCdsaWOKBZU3uuGIbtr6OjFMCMiM6ELH9N6HHRKA923", + "xJP+yP49nQX0kUQE0puZbDI+XbnJr9PiGpfXs3ATrNvZ76rz/apaQwmXVMj4n3mwIRIc3j8B5z5dXuF/", + "xF4f+tRiwl5aYJe2J5vuwOQlCoIm+ZlnpZ911grs5HNqQj2lKwude+xUI9p1IYW6CtOVpEGBA3Sa4zCT", + "bJiC8Y9LyZb8Gug3O2hUkZxZGOxLYe6v9HEpm1/+aMNUa83E1A1rPgsEcWa72u6J6d1k4QZ43Jq7g6m7", + "2nRXpPdq37Hh6F1N/ZN2am6ZDY7RW6VBXigk5DjkvQnhUe3xSSWAhWFa2uLnZSt+9pzxWp+7PAg3GsL9", + "c22cDRZPkuKm2didmv4EE2FEk/kzhm4J3wfY1gRhlJqOFoiK1t/QrzE4IWVLQsc8PLzpu3qw/mb6zn8j", + "foEbMg5kjt1mJzRnqw08DYq4kSnufG3C/1jSMK2sl96aVw5mWYY/EfULbN77aC4NRLDWHNgFa3uL5vW9", + "W91gtjrW9ug3LD4/C+oOYLYz4NHbAHTBbGgst3nuRiP/fyvjKFZpBI9S59jhnkW4AJjDHAxrrhhQfPbc", + "8JH+VbI/e/QqaXs+fnzU18RKrlmKeYKrDNYv/meC2u3ZJnujuZDmTsdA4FRNc2k1wbC6AK2AgbQeTjtC", + "GyPTa4tn4BJ47s/SAgv/fUxoxk+5Gj/DfY5fegBJhIed2+AxHmojFkJx6XEi/Ak7mYcZXam7M73N3Rfh", + "mllbocLdFdF12X7strkPxxXjwq/IIa3oAuwOe3sRCBs/D4Qlt6nlzrW2zhy4sXUGeNH3EE05nwqF9j0s", + "6If5POGwGxfmfmWl7dVrUGfvzx7dtTyoY08RO+fyh3tHUQgmvI5FQMFdtmQpuBUEZQ/s7Ey71CMwYQ6A", + "CPCX1rQZ+J0mYa512Zc4DyLVGRlxuHN8h9XWFthaTlC80ugMrBdECvhigz9d9+yO0onLrSZ0zFBmlzT9", + "Rd6ly46wk88lAvnIEBps2+MO+177DgV3w4fePufaZCKVa5ZJbamX8fX5+SnLtFLgf/2CHFjdxgmOdy6U", + "sEuwPXkBg7c8c8zyAkIa6bSf/8dXcl1hhkcv2MkrVUv1C39XnKwp6EIKMQmwVOfrraG025dBFG1pMWRL", + "aPTgZwqoNAQ7TToHU4OfLeyPIQ3G6oSzIOeT1p/5YZuh6/1Gp/W5qW/g/FyBEWBHnVG70cbk0qQ332Uj", + "QJ+cnvSH/brHZrooKhVucKBLH86KNuBD/ykS64l/T05PRh6RV7lW+GFDvsWCf7/RaVPI2g78IK+b1zf/", + "FwAA//9HZfvdYFcAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/pkg/api/openapi_types.gen.go b/pkg/api/openapi_types.gen.go index 227485c0..498d5939 100644 --- a/pkg/api/openapi_types.gen.go +++ b/pkg/api/openapi_types.gen.go @@ -267,11 +267,16 @@ type ShamanFileSpec struct { Size int `json:"size"` } -// ShamanFileSpecWithStatus defines model for ShamanFileSpecWithStatus. +// Specification of a file, which could be in the Shaman storage, or not, depending on its status. type ShamanFileSpecWithStatus struct { - // Embedded struct due to allOf(#/components/schemas/ShamanFileSpec) - ShamanFileSpec `yaml:",inline"` - // Embedded fields due to inline allOf schema + // Location of the file in the checkout + Path string `json:"path"` + + // SHA256 checksum of the file + Sha string `json:"sha"` + + // File size in bytes + Size int `json:"size"` Status ShamanFileStatus `json:"status"` } diff --git a/pkg/shaman/checkout/checkout.go b/pkg/shaman/checkout/checkout.go index ad14dcba..bdce558b 100644 --- a/pkg/shaman/checkout/checkout.go +++ b/pkg/shaman/checkout/checkout.go @@ -6,6 +6,8 @@ import ( "context" "errors" "fmt" + "regexp" + "strings" "git.blender.org/flamenco/pkg/api" "git.blender.org/flamenco/pkg/shaman/filestore" @@ -14,6 +16,8 @@ import ( var ( ErrMissingFiles = errors.New("unknown files requested in checkout") + + validCheckoutRegexp = regexp.MustCompile(`^[^/?*:;{}\\][^?*:;{}\\]*$`) ) func (m *Manager) Checkout(ctx context.Context, checkout api.ShamanCheckout) error { @@ -54,3 +58,13 @@ func (m *Manager) Checkout(ctx context.Context, checkout api.ShamanCheckout) err logger.Info().Msg("shaman: checkout created") return nil } + +func isValidCheckoutPath(checkoutPath string) bool { + if !validCheckoutRegexp.MatchString(checkoutPath) { + return false + } + if strings.Contains(checkoutPath, "../") || strings.Contains(checkoutPath, "/..") { + return false + } + return true +} diff --git a/pkg/shaman/checkout/checkout_id.go b/pkg/shaman/checkout/checkout_id.go deleted file mode 100644 index fc91c8fd..00000000 --- a/pkg/shaman/checkout/checkout_id.go +++ /dev/null @@ -1,31 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package checkout - -import "regexp" - -var validCheckoutRegexp = regexp.MustCompile("^[a-zA-Z0-9_ /]+$") - -func isValidCheckoutPath(checkoutID string) bool { - return validCheckoutRegexp.MatchString(checkoutID) -} diff --git a/pkg/shaman/checkout/checkout_test.go b/pkg/shaman/checkout/checkout_test.go new file mode 100644 index 00000000..e8884560 --- /dev/null +++ b/pkg/shaman/checkout/checkout_test.go @@ -0,0 +1,38 @@ +package checkout + +import ( + "testing" +) + +func Test_isValidCheckoutPath(t *testing.T) { + tests := []struct { + name string + checkoutPath string + want bool + }{ + // Valid cases. + {"simple", "a", true}, + {"uuid", "5e5be786-e6d7-480c-90e6-437f9ef5bf5d", true}, + {"with-spaces", "5e5be786 e6d7 480c 90e6 437f9ef5bf5d", true}, + {"project-scene-job-discriminator", "Sprite-Fright/scenename/jobname/2022-03-25-11-30-feb3", true}, + {"unicode", "ránið/lélegt vélmenni", true}, + + // Invalid cases. + {"empty", "", false}, + {"backslashes", "with\\backslash", false}, + {"windows-drive-letter", "c:/blah", false}, + {"question-mark", "blah?", false}, + {"star", "blah*hi", false}, + {"semicolon", "blah;hi", false}, + {"colon", "blah:hi", false}, + {"absolute-path", "/blah", false}, + {"directory-up", "path/../../../../etc/passwd", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isValidCheckoutPath(tt.checkoutPath); got != tt.want { + t.Errorf("isValidCheckoutPath() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/shaman/checkout/manager.go b/pkg/shaman/checkout/manager.go index fbe68aac..b2726dc3 100644 --- a/pkg/shaman/checkout/manager.go +++ b/pkg/shaman/checkout/manager.go @@ -24,6 +24,7 @@ package checkout import ( "errors" + "fmt" "os" "path" "path/filepath" @@ -54,10 +55,17 @@ type ResolvedCheckoutInfo struct { RelativePath string } +type ErrInvalidCheckoutPath struct { + CheckoutPath string +} + +func (err ErrInvalidCheckoutPath) Error() string { + return fmt.Sprintf("invalid checkout path %q", err.CheckoutPath) +} + // Errors returned by the Checkout Manager. var ( ErrCheckoutAlreadyExists = errors.New("A checkout with this ID already exists") - ErrInvalidCheckoutID = errors.New("The Checkout ID is invalid") ) // NewManager creates and returns a new Checkout Manager. @@ -81,7 +89,7 @@ func (m *Manager) Close() { func (m *Manager) pathForCheckout(requestedCheckoutPath string) (ResolvedCheckoutInfo, error) { if !isValidCheckoutPath(requestedCheckoutPath) { - return ResolvedCheckoutInfo{}, ErrInvalidCheckoutID + return ResolvedCheckoutInfo{}, ErrInvalidCheckoutPath{requestedCheckoutPath} } return ResolvedCheckoutInfo{ @@ -219,5 +227,5 @@ func touchFile(blobPath string) error { } logger.Debug().Msg("done touching") - return err + return nil } diff --git a/pkg/shaman/checkout/report_requirements.go b/pkg/shaman/checkout/report_requirements.go index 84201284..2cca22e6 100644 --- a/pkg/shaman/checkout/report_requirements.go +++ b/pkg/shaman/checkout/report_requirements.go @@ -41,8 +41,11 @@ func (m *Manager) ReportRequirements(ctx context.Context, requirements api.Shama case filestore.StatusStored: // We expect this file to be sent soon, though, so we need to // 'touch' it to make sure it won't be GC'd in the mean time. - go touchFile(storePath) - + go func() { + if err := touchFile(storePath); err != nil { + logger.Error().Err(err).Str("path", storePath).Msg("shaman: error touching file") + } + }() // Only send a response when the caller needs to do something. continue default: @@ -51,15 +54,19 @@ func (m *Manager) ReportRequirements(ctx context.Context, requirements api.Shama Str("status", status.String()). Str("checksum", fileSpec.Sha). Int("filesize", fileSpec.Size). - Msg("invalid status returned by ResolveFile") + Msg("shaman: invalid status returned by ResolveFile, ignoring this file") continue } alreadyRequested[fileKey] = true - missing.Files = append(missing.Files, api.ShamanFileSpecWithStatus{ - ShamanFileSpec: fileSpec, - Status: apiStatus, - }) + fileSpec := api.ShamanFileSpecWithStatus{ + Path: fileSpec.Path, + Sha: fileSpec.Sha, + Size: fileSpec.Size, + Status: apiStatus, + } + logger.Trace().Interface("fileSpec", fileSpec).Msg("shaman: file needed from client") + missing.Files = append(missing.Files, fileSpec) } return missing, nil diff --git a/pkg/shaman/checkout/routes_test.go b/pkg/shaman/checkout/routes_test.go index 1db9a71f..6ad180c1 100644 --- a/pkg/shaman/checkout/routes_test.go +++ b/pkg/shaman/checkout/routes_test.go @@ -49,9 +49,9 @@ func TestReportRequirements(t *testing.T) { // We should not be required to upload the same file twice, so the duplicate // should not be in the response. assert.Equal(t, []api.ShamanFileSpecWithStatus{ - {ShamanFileSpec: spec1, Status: api.ShamanFileStatusUnknown}, - {ShamanFileSpec: spec2, Status: api.ShamanFileStatusUnknown}, - {ShamanFileSpec: spec3, Status: api.ShamanFileStatusUnknown}, + {Sha: spec1.Sha, Size: spec1.Size, Path: spec1.Path, Status: api.ShamanFileStatusUnknown}, + {Sha: spec2.Sha, Size: spec2.Size, Path: spec2.Path, Status: api.ShamanFileStatusUnknown}, + {Sha: spec3.Sha, Size: spec3.Size, Path: spec3.Path, Status: api.ShamanFileStatusUnknown}, }, response.Files) }