diff --git a/addon/flamenco/bat/shaman.py b/addon/flamenco/bat/shaman.py index 8b1cf13b..859cd175 100644 --- a/addon/flamenco/bat/shaman.py +++ b/addon/flamenco/bat/shaman.py @@ -19,14 +19,14 @@ if TYPE_CHECKING: from ..manager import ApiClient as _ApiClient from ..manager.models import ( + ShamanCheckoutResult as _ShamanCheckoutResult, ShamanRequirementsRequest as _ShamanRequirementsRequest, - ShamanRequirementsResponse as _ShamanRequirementsResponse, ShamanFileSpec as _ShamanFileSpec, ) else: _ApiClient = object + _ShamanCheckoutResult = object _ShamanRequirementsRequest = object - _ShamanRequirementsResponse = object _ShamanFileSpec = object log = logging.getLogger(__name__) @@ -58,10 +58,14 @@ class Packer(bat_pack.Packer): # type: ignore super().__init__(blendfile, project_root, target, **kwargs) self.checkout_path = checkout_path self.api_client = api_client + self.shaman_transferrer: Optional[Transferrer] = None # Mypy doesn't understand that bat_transfer.FileTransferer exists. def _create_file_transferer(self) -> bat_transfer.FileTransferer: # type: ignore - return Transferrer(self.api_client, self.project, self.checkout_path) + self.shaman_transferrer = Transferrer( + self.api_client, self.project, self.checkout_path + ) + return self.shaman_transferrer def _make_target_path(self, target: str) -> PurePath: return PurePosixPath("/") @@ -70,10 +74,11 @@ class Packer(bat_pack.Packer): # type: ignore def output_path(self) -> PurePath: """The path of the packed blend file in the target directory.""" assert self._output_path is not None + assert self.shaman_transferrer is not None - checkout_location = PurePosixPath(self.checkout_path) + checkout_root = PurePosixPath(self.shaman_transferrer.checkout_path) rel_output = self._output_path.relative_to(self._target_path) - out_path: PurePath = checkout_location / rel_output + out_path: PurePath = checkout_root / rel_output return out_path def execute(self): @@ -149,7 +154,10 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore return self.log.info("All files uploaded succesfully") - self._request_checkout(shaman_file_specs) + checkout_result = self._request_checkout(shaman_file_specs) + + # Update our checkout path to match the one received from the Manager. + self.checkout_path = checkout_result.checkout_path def _upload_missing_files( self, shaman_file_specs: _ShamanRequirementsRequest @@ -407,14 +415,16 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore raise self.AbortUpload("interrupting ongoing upload") super().report_transferred(bytes_transferred) - def _request_checkout(self, shaman_file_specs: _ShamanRequirementsRequest) -> None: + def _request_checkout( + self, shaman_file_specs: _ShamanRequirementsRequest + ) -> Optional[_ShamanCheckoutResult]: """Ask the Shaman to create a checkout of this BAT pack.""" if not self.checkout_path: self.log.warning("NOT requesting checkout at Shaman") - return + return None - from ..manager.models import ShamanCheckout + from ..manager.models import ShamanCheckout, ShamanCheckoutResult from ..manager.exceptions import ApiException self.log.info( @@ -427,7 +437,9 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore ) try: - self.shaman_api.shaman_checkout(checkoutRequest) + result: ShamanCheckoutResult = self.shaman_api.shaman_checkout( + checkoutRequest + ) except ApiException as ex: match ex.status: case 424: # Files were missing @@ -444,9 +456,10 @@ class Transferrer(bat_transfer.FileTransferer): # type: ignore ) self.log.error(msg) self.error_set(msg) - return + return None - self.log.info("Shaman created checkout at %s", self.checkout_path) + self.log.info("Shaman created checkout at %s", result.checkout_path) + return result def make_file_spec_hashable(spec: _ShamanFileSpec) -> HashableShamanFileSpec: diff --git a/addon/flamenco/manager/__init__.py b/addon/flamenco/manager/__init__.py index 6b7f4cb7..5450630c 100644 --- a/addon/flamenco/manager/__init__.py +++ b/addon/flamenco/manager/__init__.py @@ -10,7 +10,7 @@ """ -__version__ = "8a97cf50-dirty" +__version__ = "f66594b0-dirty" # import ApiClient from flamenco.manager.api_client import ApiClient diff --git a/addon/flamenco/manager/api/shaman_api.py b/addon/flamenco/manager/api/shaman_api.py index f4d6c5f8..6d5c3698 100644 --- a/addon/flamenco/manager/api/shaman_api.py +++ b/addon/flamenco/manager/api/shaman_api.py @@ -23,6 +23,7 @@ from flamenco.manager.model_utils import ( # noqa: F401 ) from flamenco.manager.model.error import Error from flamenco.manager.model.shaman_checkout import ShamanCheckout +from flamenco.manager.model.shaman_checkout_result import ShamanCheckoutResult from flamenco.manager.model.shaman_requirements_request import ShamanRequirementsRequest from flamenco.manager.model.shaman_requirements_response import ShamanRequirementsResponse from flamenco.manager.model.shaman_single_file_status import ShamanSingleFileStatus @@ -41,7 +42,7 @@ class ShamanApi(object): self.api_client = api_client self.shaman_checkout_endpoint = _Endpoint( settings={ - 'response_type': None, + 'response_type': (ShamanCheckoutResult,), 'auth': [], 'endpoint_path': '/shaman/checkout/create', 'operation_id': 'shaman_checkout', @@ -312,7 +313,7 @@ class ShamanApi(object): async_req (bool): execute request asynchronously Returns: - None + ShamanCheckoutResult If the method is called asynchronously, returns the request thread. """ diff --git a/addon/flamenco/manager/api_client.py b/addon/flamenco/manager/api_client.py index 289608dd..0f932366 100644 --- a/addon/flamenco/manager/api_client.py +++ b/addon/flamenco/manager/api_client.py @@ -76,7 +76,7 @@ class ApiClient(object): self.default_headers[header_name] = header_value self.cookie = cookie # Set default User-Agent. - self.user_agent = 'Flamenco/8a97cf50-dirty (Blender add-on)' + self.user_agent = 'Flamenco/f66594b0-dirty (Blender add-on)' def __enter__(self): return self diff --git a/addon/flamenco/manager/configuration.py b/addon/flamenco/manager/configuration.py index 62a36c63..7962c323 100644 --- a/addon/flamenco/manager/configuration.py +++ b/addon/flamenco/manager/configuration.py @@ -404,7 +404,7 @@ conf = flamenco.manager.Configuration( "OS: {env}\n"\ "Python Version: {pyversion}\n"\ "Version of the API: 1.0.0\n"\ - "SDK Package Version: 8a97cf50-dirty".\ + "SDK Package Version: f66594b0-dirty".\ format(env=sys.platform, pyversion=sys.version) def get_host_settings(self): diff --git a/addon/flamenco/manager/docs/ShamanApi.md b/addon/flamenco/manager/docs/ShamanApi.md index 2e030bd5..ce5e9dba 100644 --- a/addon/flamenco/manager/docs/ShamanApi.md +++ b/addon/flamenco/manager/docs/ShamanApi.md @@ -11,7 +11,7 @@ Method | HTTP request | Description # **shaman_checkout** -> shaman_checkout(shaman_checkout) +> ShamanCheckoutResult shaman_checkout(shaman_checkout) Create a directory, and symlink the required files into it. The files must all have been uploaded to Shaman before calling this endpoint. @@ -24,6 +24,7 @@ import flamenco.manager from flamenco.manager.api import shaman_api from flamenco.manager.model.error import Error from flamenco.manager.model.shaman_checkout import ShamanCheckout +from flamenco.manager.model.shaman_checkout_result import ShamanCheckoutResult from pprint import pprint # Defining the host is optional and defaults to http://localhost # See configuration.py for a list of all supported configuration parameters. @@ -50,7 +51,8 @@ with flamenco.manager.ApiClient() as api_client: # example passing only required values which don't have defaults set try: # Create a directory, and symlink the required files into it. The files must all have been uploaded to Shaman before calling this endpoint. - api_instance.shaman_checkout(shaman_checkout) + api_response = api_instance.shaman_checkout(shaman_checkout) + pprint(api_response) except flamenco.manager.ApiException as e: print("Exception when calling ShamanApi->shaman_checkout: %s\n" % e) ``` @@ -64,7 +66,7 @@ Name | Type | Description | Notes ### Return type -void (empty response body) +[**ShamanCheckoutResult**](ShamanCheckoutResult.md) ### Authorization @@ -80,7 +82,7 @@ No authorization required | Status code | Description | Response headers | |-------------|-------------|------------------| -**204** | Checkout was created succesfully. | - | +**200** | Checkout was created succesfully. | - | **424** | There were files missing. Use `shamanCheckoutRequirements` to figure out which ones. | - | **409** | Checkout already exists. | - | **0** | unexpected error | - | diff --git a/addon/flamenco/manager/docs/ShamanCheckout.md b/addon/flamenco/manager/docs/ShamanCheckout.md index ac16162d..de467935 100644 --- a/addon/flamenco/manager/docs/ShamanCheckout.md +++ b/addon/flamenco/manager/docs/ShamanCheckout.md @@ -6,7 +6,7 @@ Set of files with their SHA256 checksum, size in bytes, and desired location in Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **files** | [**[ShamanFileSpec]**](ShamanFileSpec.md) | | -**checkout_path** | **str** | Path where the Manager should create this checkout, It is relative to the Shaman checkout path as configured on the Manager. In older versions of the Shaman this was just the \"checkout ID\", but in this version it can be a path like `project-slug/scene-name/unique-ID`. | +**checkout_path** | **str** | Path where the Manager should create this checkout. It is relative to the Shaman checkout path as configured on the Manager. In older versions of the Shaman this was just the \"checkout ID\", but in this version it can be a path like `project-slug/scene-name/unique-ID`. | **any string name** | **bool, date, datetime, dict, float, int, list, str, none_type** | any string name can be used but the value must be the correct type | [optional] [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/addon/flamenco/manager/docs/ShamanCheckoutResult.md b/addon/flamenco/manager/docs/ShamanCheckoutResult.md new file mode 100644 index 00000000..3def54a7 --- /dev/null +++ b/addon/flamenco/manager/docs/ShamanCheckoutResult.md @@ -0,0 +1,13 @@ +# ShamanCheckoutResult + +The result of a Shaman checkout. + +## Properties +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**checkout_path** | **str** | Path where the Manager created this checkout. This can be different than what was requested, as the Manager will ensure a unique directory. The path is relative to the Shaman checkout path as configured on the Manager. | +**any string name** | **bool, date, datetime, dict, float, int, list, str, none_type** | any string name can be used but the value must be the correct type | [optional] + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/addon/flamenco/manager/model/shaman_checkout.py b/addon/flamenco/manager/model/shaman_checkout.py index af690b2e..1341b12d 100644 --- a/addon/flamenco/manager/model/shaman_checkout.py +++ b/addon/flamenco/manager/model/shaman_checkout.py @@ -113,7 +113,7 @@ class ShamanCheckout(ModelNormal): Args: files ([ShamanFileSpec]): - checkout_path (str): Path where the Manager should create this checkout, It is relative to the Shaman checkout path as configured on the Manager. In older versions of the Shaman this was just the \"checkout ID\", but in this version it can be a path like `project-slug/scene-name/unique-ID`. + checkout_path (str): Path where the Manager should create this checkout. It is relative to the Shaman checkout path as configured on the Manager. In older versions of the Shaman this was just the \"checkout ID\", but in this version it can be a path like `project-slug/scene-name/unique-ID`. Keyword Args: _check_type (bool): if True, values for parameters in openapi_types @@ -200,7 +200,7 @@ class ShamanCheckout(ModelNormal): Args: files ([ShamanFileSpec]): - checkout_path (str): Path where the Manager should create this checkout, It is relative to the Shaman checkout path as configured on the Manager. In older versions of the Shaman this was just the \"checkout ID\", but in this version it can be a path like `project-slug/scene-name/unique-ID`. + checkout_path (str): Path where the Manager should create this checkout. It is relative to the Shaman checkout path as configured on the Manager. In older versions of the Shaman this was just the \"checkout ID\", but in this version it can be a path like `project-slug/scene-name/unique-ID`. Keyword Args: _check_type (bool): if True, values for parameters in openapi_types diff --git a/addon/flamenco/manager/model/shaman_checkout_result.py b/addon/flamenco/manager/model/shaman_checkout_result.py new file mode 100644 index 00000000..bd15488e --- /dev/null +++ b/addon/flamenco/manager/model/shaman_checkout_result.py @@ -0,0 +1,261 @@ +""" + Flamenco manager + + Render Farm manager API # noqa: E501 + + The version of the OpenAPI document: 1.0.0 + Generated by: https://openapi-generator.tech +""" + + +import re # noqa: F401 +import sys # noqa: F401 + +from flamenco.manager.model_utils import ( # noqa: F401 + ApiTypeError, + ModelComposed, + ModelNormal, + ModelSimple, + cached_property, + change_keys_js_to_python, + convert_js_args_to_python_args, + date, + datetime, + file_type, + none_type, + validate_get_composed_info, + OpenApiModel +) +from flamenco.manager.exceptions import ApiAttributeError + + + +class ShamanCheckoutResult(ModelNormal): + """NOTE: This class is auto generated by OpenAPI Generator. + Ref: https://openapi-generator.tech + + Do not edit the class manually. + + Attributes: + allowed_values (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + with a capitalized key describing the allowed value and an allowed + value. These dicts store the allowed enum values. + attribute_map (dict): The key is attribute name + and the value is json key in definition. + discriminator_value_class_map (dict): A dict to go from the discriminator + variable value to the discriminator class name. + validations (dict): The key is the tuple path to the attribute + and the for var_name this is (var_name,). The value is a dict + that stores validations for max_length, min_length, max_items, + min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum, + inclusive_minimum, and regex. + additional_properties_type (tuple): A tuple of classes accepted + as additional properties values. + """ + + allowed_values = { + } + + validations = { + } + + @cached_property + def additional_properties_type(): + """ + This must be a method because a model may have properties that are + of type self, this must run after the class is loaded + """ + return (bool, date, datetime, dict, float, int, list, str, none_type,) # noqa: E501 + + _nullable = False + + @cached_property + def openapi_types(): + """ + This must be a method because a model may have properties that are + of type self, this must run after the class is loaded + + Returns + openapi_types (dict): The key is attribute name + and the value is attribute type. + """ + return { + 'checkout_path': (str,), # noqa: E501 + } + + @cached_property + def discriminator(): + return None + + + attribute_map = { + 'checkout_path': 'checkoutPath', # noqa: E501 + } + + read_only_vars = { + } + + _composed_schemas = {} + + @classmethod + @convert_js_args_to_python_args + def _from_openapi_data(cls, checkout_path, *args, **kwargs): # noqa: E501 + """ShamanCheckoutResult - a model defined in OpenAPI + + Args: + checkout_path (str): Path where the Manager created this checkout. This can be different than what was requested, as the Manager will ensure a unique directory. The path is relative to the Shaman checkout path as configured on the Manager. + + Keyword Args: + _check_type (bool): if True, values for parameters in openapi_types + will be type checked and a TypeError will be + raised if the wrong type is input. + Defaults to True + _path_to_item (tuple/list): This is a list of keys or values to + drill down to the model in received_data + when deserializing a response + _spec_property_naming (bool): True if the variable names in the input data + are serialized names, as specified in the OpenAPI document. + False if the variable names in the input data + are pythonic names, e.g. snake case (default) + _configuration (Configuration): the instance to use when + deserializing a file_type parameter. + If passed, type conversion is attempted + If omitted no type conversion is done. + _visited_composed_classes (tuple): This stores a tuple of + classes that we have traveled through so that + if we see that class again we will not use its + discriminator again. + When traveling through a discriminator, the + composed schema that is + is traveled through is added to this set. + For example if Animal has a discriminator + petType and we pass in "Dog", and the class Dog + allOf includes Animal, we move through Animal + once using the discriminator, and pick Dog. + Then in Dog, we will make an instance of the + Animal class but this time we won't travel + through its discriminator because we passed in + _visited_composed_classes = (Animal,) + """ + + _check_type = kwargs.pop('_check_type', True) + _spec_property_naming = kwargs.pop('_spec_property_naming', False) + _path_to_item = kwargs.pop('_path_to_item', ()) + _configuration = kwargs.pop('_configuration', None) + _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) + + self = super(OpenApiModel, cls).__new__(cls) + + if args: + raise ApiTypeError( + "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( + args, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + self._data_store = {} + self._check_type = _check_type + self._spec_property_naming = _spec_property_naming + self._path_to_item = _path_to_item + self._configuration = _configuration + self._visited_composed_classes = _visited_composed_classes + (self.__class__,) + + self.checkout_path = checkout_path + for var_name, var_value in kwargs.items(): + if var_name not in self.attribute_map and \ + self._configuration is not None and \ + self._configuration.discard_unknown_keys and \ + self.additional_properties_type is None: + # discard variable. + continue + setattr(self, var_name, var_value) + return self + + required_properties = set([ + '_data_store', + '_check_type', + '_spec_property_naming', + '_path_to_item', + '_configuration', + '_visited_composed_classes', + ]) + + @convert_js_args_to_python_args + def __init__(self, checkout_path, *args, **kwargs): # noqa: E501 + """ShamanCheckoutResult - a model defined in OpenAPI + + Args: + checkout_path (str): Path where the Manager created this checkout. This can be different than what was requested, as the Manager will ensure a unique directory. The path is relative to the Shaman checkout path as configured on the Manager. + + Keyword Args: + _check_type (bool): if True, values for parameters in openapi_types + will be type checked and a TypeError will be + raised if the wrong type is input. + Defaults to True + _path_to_item (tuple/list): This is a list of keys or values to + drill down to the model in received_data + when deserializing a response + _spec_property_naming (bool): True if the variable names in the input data + are serialized names, as specified in the OpenAPI document. + False if the variable names in the input data + are pythonic names, e.g. snake case (default) + _configuration (Configuration): the instance to use when + deserializing a file_type parameter. + If passed, type conversion is attempted + If omitted no type conversion is done. + _visited_composed_classes (tuple): This stores a tuple of + classes that we have traveled through so that + if we see that class again we will not use its + discriminator again. + When traveling through a discriminator, the + composed schema that is + is traveled through is added to this set. + For example if Animal has a discriminator + petType and we pass in "Dog", and the class Dog + allOf includes Animal, we move through Animal + once using the discriminator, and pick Dog. + Then in Dog, we will make an instance of the + Animal class but this time we won't travel + through its discriminator because we passed in + _visited_composed_classes = (Animal,) + """ + + _check_type = kwargs.pop('_check_type', True) + _spec_property_naming = kwargs.pop('_spec_property_naming', False) + _path_to_item = kwargs.pop('_path_to_item', ()) + _configuration = kwargs.pop('_configuration', None) + _visited_composed_classes = kwargs.pop('_visited_composed_classes', ()) + + if args: + raise ApiTypeError( + "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % ( + args, + self.__class__.__name__, + ), + path_to_item=_path_to_item, + valid_classes=(self.__class__,), + ) + + self._data_store = {} + self._check_type = _check_type + self._spec_property_naming = _spec_property_naming + self._path_to_item = _path_to_item + self._configuration = _configuration + self._visited_composed_classes = _visited_composed_classes + (self.__class__,) + + self.checkout_path = checkout_path + for var_name, var_value in kwargs.items(): + if var_name not in self.attribute_map and \ + self._configuration is not None and \ + self._configuration.discard_unknown_keys and \ + self.additional_properties_type is None: + # discard variable. + continue + setattr(self, var_name, var_value) + if var_name in self.read_only_vars: + raise ApiAttributeError(f"`{var_name}` is a read-only attribute. Use `from_openapi_data` to instantiate " + f"class with read only attributes.") diff --git a/addon/flamenco/manager/models/__init__.py b/addon/flamenco/manager/models/__init__.py index e97d5117..b887f0f3 100644 --- a/addon/flamenco/manager/models/__init__.py +++ b/addon/flamenco/manager/models/__init__.py @@ -27,6 +27,7 @@ from flamenco.manager.model.manager_configuration import ManagerConfiguration from flamenco.manager.model.registered_worker import RegisteredWorker from flamenco.manager.model.security_error import SecurityError from flamenco.manager.model.shaman_checkout import ShamanCheckout +from flamenco.manager.model.shaman_checkout_result import ShamanCheckoutResult from flamenco.manager.model.shaman_file_spec import ShamanFileSpec from flamenco.manager.model.shaman_file_spec_with_status import ShamanFileSpecWithStatus from flamenco.manager.model.shaman_file_status import ShamanFileStatus diff --git a/addon/flamenco/manager_README.md b/addon/flamenco/manager_README.md index 0df9b5ca..b6d77649 100644 --- a/addon/flamenco/manager_README.md +++ b/addon/flamenco/manager_README.md @@ -4,7 +4,7 @@ Render Farm manager API The `flamenco.manager` package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project: - API version: 1.0.0 -- Package version: 8a97cf50-dirty +- Package version: f66594b0-dirty - Build package: org.openapitools.codegen.languages.PythonClientCodegen For more information, please visit [https://flamenco.io/](https://flamenco.io/) @@ -102,6 +102,7 @@ Class | Method | HTTP request | Description - [RegisteredWorker](flamenco/manager/docs/RegisteredWorker.md) - [SecurityError](flamenco/manager/docs/SecurityError.md) - [ShamanCheckout](flamenco/manager/docs/ShamanCheckout.md) + - [ShamanCheckoutResult](flamenco/manager/docs/ShamanCheckoutResult.md) - [ShamanFileSpec](flamenco/manager/docs/ShamanFileSpec.md) - [ShamanFileSpecWithStatus](flamenco/manager/docs/ShamanFileSpecWithStatus.md) - [ShamanFileStatus](flamenco/manager/docs/ShamanFileStatus.md) diff --git a/addon/flamenco/operators.py b/addon/flamenco/operators.py index c3cd7e50..093173a0 100644 --- a/addon/flamenco/operators.py +++ b/addon/flamenco/operators.py @@ -291,9 +291,8 @@ class FLAMENCO_OT_submit_job(FlamencoOpMixin, bpy.types.Operator): assert self.job is not None self.log.info("Sending BAT pack to Shaman") - # TODO: get project name from preferences/GUI. - # TODO: update Shaman API to ensure this is a unique location. - checkout_root = PurePosixPath(f"project/{self.job.name}") + # TODO: get project name from preferences/GUI and insert that here too. + checkout_root = PurePosixPath(f"{self.job.name}") self.packthread = bat_interface.copy( base_blendfile=blendfile, diff --git a/internal/manager/api_impl/api_impl.go b/internal/manager/api_impl/api_impl.go index a3e030c0..12aacd13 100644 --- a/internal/manager/api_impl/api_impl.go +++ b/internal/manager/api_impl/api_impl.go @@ -84,7 +84,8 @@ type Shaman interface { // Checkout creates a directory, and symlinks the required files into it. The // files must all have been uploaded to Shaman before calling this. - Checkout(ctx context.Context, checkout api.ShamanCheckout) error + // Returns the final checkout directory, as it may be modified to ensure uniqueness. + Checkout(ctx context.Context, checkout api.ShamanCheckout) (string, error) // Requirements checks a Shaman Requirements file, and returns the subset // containing the unknown files. diff --git a/internal/manager/api_impl/mocks/api_impl_mock.gen.go b/internal/manager/api_impl/mocks/api_impl_mock.gen.go index c74a8b9f..0a8f3de2 100644 --- a/internal/manager/api_impl/mocks/api_impl_mock.gen.go +++ b/internal/manager/api_impl/mocks/api_impl_mock.gen.go @@ -412,11 +412,12 @@ func (m *MockShaman) EXPECT() *MockShamanMockRecorder { } // Checkout mocks base method. -func (m *MockShaman) Checkout(arg0 context.Context, arg1 api.ShamanCheckout) error { +func (m *MockShaman) Checkout(arg0 context.Context, arg1 api.ShamanCheckout) (string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Checkout", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 } // Checkout indicates an expected call of Checkout. diff --git a/internal/manager/api_impl/shaman.go b/internal/manager/api_impl/shaman.go index bd24ac91..2e972c18 100644 --- a/internal/manager/api_impl/shaman.go +++ b/internal/manager/api_impl/shaman.go @@ -31,14 +31,16 @@ func (f *Flamenco) ShamanCheckout(e echo.Context) error { return sendAPIError(e, http.StatusBadRequest, "invalid format") } - err = f.shaman.Checkout(e.Request().Context(), api.ShamanCheckout(reqBody)) + checkoutPath, err := f.shaman.Checkout(e.Request().Context(), api.ShamanCheckout(reqBody)) if err != nil { // TODO: return 409 when checkout already exists. logger.Warn().Err(err).Msg("Shaman: creating checkout") return sendAPIError(e, http.StatusInternalServerError, "unexpected error: %v", err) } - return e.String(http.StatusNoContent, "") + return e.JSON(http.StatusOK, api.ShamanCheckoutResult{ + CheckoutPath: checkoutPath, + }) } // Checks a Shaman Requirements file, and reports which files are unknown. diff --git a/pkg/api/flamenco-manager.yaml b/pkg/api/flamenco-manager.yaml index dae1353f..93e9f5a2 100644 --- a/pkg/api/flamenco-manager.yaml +++ b/pkg/api/flamenco-manager.yaml @@ -313,8 +313,12 @@ paths: schema: $ref: "#/components/schemas/ShamanCheckout" responses: - "204": + "200": description: Checkout was created succesfully. + content: + application/json: + schema: + $ref: '#/components/schemas/ShamanCheckoutResult' "424": description: There were files missing. Use `shamanCheckoutRequirements` to figure out which ones. content: @@ -795,6 +799,19 @@ components: size: 127 path: logging.go + ShamanCheckoutResult: + type: object + description: The result of a Shaman checkout. + properties: + "checkoutPath": + type: string + description: > + Path where the Manager created this checkout. This can be different + than what was requested, as the Manager will ensure a unique + directory. The path is relative to the Shaman checkout path as + configured on the Manager. + required: [checkoutPath] + ShamanFileStatus: type: string enum: [unknown, uploading, stored] diff --git a/pkg/api/openapi_client.gen.go b/pkg/api/openapi_client.gen.go index 1adbc70c..a8fbf3a5 100644 --- a/pkg/api/openapi_client.gen.go +++ b/pkg/api/openapi_client.gen.go @@ -1397,6 +1397,7 @@ func (r TaskUpdateResponse) StatusCode() int { type ShamanCheckoutResponse struct { Body []byte HTTPResponse *http.Response + JSON200 *ShamanCheckoutResult JSON409 *Error JSON424 *Error JSONDefault *Error @@ -2054,6 +2055,13 @@ func ParseShamanCheckoutResponse(rsp *http.Response) (*ShamanCheckoutResponse, e } switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest ShamanCheckoutResult + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { diff --git a/pkg/api/openapi_spec.gen.go b/pkg/api/openapi_spec.gen.go index 7ea2f44c..f00bd9f8 100644 --- a/pkg/api/openapi_spec.gen.go +++ b/pkg/api/openapi_spec.gen.go @@ -18,94 +18,95 @@ import ( // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+Q87W4cN5KvQvQekF3cfOnDlq1f57XjREESC5G8OSA2JHZ3zQwtNtkh2RpPDAH7EPcm", - "dwvcj9tf9wLeNzoUi/01zZHGieX17vmHMZpmVxXru4rFeZdkuii1AuVscvwusdkSCu4/PrFWLBTk59xe", - "4d852MyI0gmtkuPeUyYs48zhJ26ZcPi3gQzENeQsXTO3BPajNldgJskoKY0uwTgBHkumi4Kr3H8WDgr/", - "4V8MzJPj5HfTlrhpoGz6lF5IbkaJW5eQHCfcGL7Gv9/oFN8OX1tnhFqE7y9KI7QRbt1ZIJSDBZh6BX0b", - "eV3xIv7gdpjWcVfduR3k3xmtxB1xe7WdkKoSOT6Ya1NwlxzTF6PNhTejxMDPlTCQJ8c/1YuQOWEvDW2d", - "LWxwqcOSLlWjVl6vG7w6fQOZQwKfXHMheSrhG52egXNIzkBzzoRaSGCWnjM9Z5x9o1OG0GxEQZZaZPSx", - "D+fHJSi2ENegRkyKQjivZ9dcihz/r8Ayp/E7CywAmbAXSq5ZZZFGthJuyYhpHjniblRwwPxNZcthzivp", - "hnSdL4GFh0QHs0u9UoEYVlkwbIW05+DAFEJ5/Etha5ZMCHwHZhxF883UaS2dKAMioVpEqI9mzjPwQCEX", - "DrdOEAP9cy4tjIbMdUswSDSXUq8YvrpJKONzh2uWwN7olC25ZSmAYrZKC+Ec5BP2o65kzkRRyjXLQQK9", - "JiWDt8ISQG6vLJtrQ6Df6HTEuMrRgeiiFBLXCDd5pVpFT7WWwJXf0TWXQ/6crt1SKwZvSwPWCu2ZnwLD", - "1RV3kCOPtMlpg7UcwO+kL7qGrkY2o6FqXMF6SMNJDsqJuQATgDQqP2JFZR3SUynxc0WKGIT2JhhCFA8a", - "BjeLiC08UWsGb53hjJtFVaCHqfUtLdcTfNFOznQBp2Rb69//gWUohspCjiszA9wBbTXY37pDQ2virWf5", - "ABUSRQG54A7kmhlAUIz7reYwF0rgCyN0BB49ohx5nujKBYq4cSKrJDeNHLbog63S2n3e5nUjjuosvNmY", - "+gdDOA+vXwsrNo3Mmeo2BqHh9k0r6MPLE3KQyKzarAz7vRRXwDj7owSFSszzfKzVHybsDByCu/QCuSQ3", - "Q/GYK/IFissGh1tyh6grmasvvEI2ngpU7h2IjTN6I8SgAYRFO4aFs1ZOG9GhSsf4hNSBDKKWOXtaGQPK", - "yTXT6Md5DddbWMeT2wm7/PrJ2ddfPrt4fvLtlxenT86/vqQsJRcGMqfNmpXcLdm/sstXyfR3/t+r5JLx", - "skSW5rRtUFWB+5sLCRe4PhkluTD1R/91iKhLbpeQX7QrX0cMeJvSDB184EBn9x2vQeGLW3byrLZnv21U", - "mqASE/a9Zgos+jrrTJW5yoBlv/fhy45YLjJExY0A+wfGDTBblaU2bnPrgfgRZjYH+7hpqblLRl4X7txk", - "fHd1tG9xUpYoLPuOK74AQyFAOG/6vEAHHUkNJE9BfljKFpi5e7oZS2kG2cCGOQSVIPI6OO+yDeRWxLl/", - "K6yrlcFr93a+DXlUp3G/bsfnPY+4ZbstitgG63x9sK3wgBnAKO1DFmeWksOQZXpP9BayysFddcT2JL1R", - "oM7jmry44DqvxHb0pTHaILDNSiaHXnZeW8ywNCjAWr6I0btBkIfZro9R81zyAlSm/wTGhmRxR85ct2/c", - "TkW9MNhVjIpvqPTiUr6YJ8c/3a5hZ3V+iG/djAaM9LlITGPwgc/mRAHW8aJEf1SzO+cOxvgkljqJCLiX", - "L0+e1WHmG18d3VFY7VrToatoSrqqzD/ybjak4ymtedbia4h9ffOaBPQdOJ5zx72g8tynXVye9ng/2PFG", - "nmlS4Qw3a1YEYCHs2gn7ThtvuKWEt92Yk3GFUavQmP97j1WhlbNLPkkn2SVT2hEf6jT5CnzqCW85wgoK", - "7RXtODkrjXDAnhuxWGIUwhxlAgUXEqlepwbUv6UhBGqzqFeQDSRnfgE7c//7P9cgO46tp8hnnRgR5xNl", - "c9F3GwWpAyjPnLj2lTNXGXKAiuhSggufFTFLaDWec0Ermg8lxxQ9GSU/V1D5D9xkS3Hd+UjxmcCPUTN8", - "2A9Ael/4zwSlQhaNu8iTUbLivsgbz7UZYyZjowE+xJqnWs3FojLcRd2OXfKCqy8VRpI8Wr1T8rsEduaX", - "MsTInOHKzsGwJ6cnPmOro9Hk7jS0jzLmpn6AhbAODOQUR4Zk8zzHmjFqC5Jbd+Hl2W/6dPIOkV1tj0SS", - "O7TveGKi527FzZasZSe3Q1tqPU+TJVw0DZx+FnBnj+M3NZwaXowapnYbTzUzRklGWb2nMtnkcoczW3YU", - "k/MZZJURbr0lVO8cf28LvKS1T5eQXekq0gfCWkzPvVZb6jW5JQjDzr5+sv/gIcvwRVsVI2bFLz51T9cO", - "LGW+OVgkgUmdkW8M5WAWsLVlzIanJKvFAOyLkOOkrbAnC40sXPLkODl4kM4OH+9l+0fp7ODgIN+bp4cP", - "5tns6NFjvref8dnDdC9/eDjL9x88fHz0aJY+mh3l8GB2mB/N9h/DDAGJXyA53jvcP/QRnLBJvVhgpdZB", - "9fAgPdrPHh6kjw/3D+f53kH6+OBoNk8fzmYPH88ezbIDvvfgaO8omx/w/PBw/+HBg3Tv0VH2kD96/GB2", - "9LhFtX/kI9lme5A4cuoJGDSCsMZbLcFQbyd4rlDz9poeNZwROwn9a8nRcddtlOCjGgH46pFblgUvCDnT", - "qotkwk4U0xLL85A/2TrZCLA83hW37A0WdvjgVbMddvLsVTJiaeVI9MLWULCeDyGVExW+GXAZYuTYymox", - "tRkoGKP1TanHND55dtkr5VujDyqzY3VAtD8XEs5KyO6sDQj4qC+m7dbUwB1aUwmZmItgDr4n4uNFMIzA", - "Uuu04QsYlgdlVD2+1S08hNKFWFMcTQOXPEJh3667MKMwvFJvAkEG9B1CpIAYhr3aSEZJuRuDfxRu2eYq", - "O7F6xFZLkS1Z5m0n3cL6EdMGU7oRy6EElfsWv/KlPPn+f3LZ7BqsO+IIAftOqbYZ/e3iHaSglbpSeqV8", - "cSA1zylhRIH10qR2/wTsB6LGd5N/oCzyV0c5H9V6vNsauO4pQn2SaPQJfOl24fflZUutLMTPjkhac6ML", - "xpnpvMZCtTDqipK0rg5vtbmDucYg99yD8p1kboB5RcOYGZbhd/A2k1UOOSFEGCZQ9yl1oDXMxh7uRy26", - "iBpz+8i60nHfv1Vr6Di27zg2TDzI/0Nj7sdyhLc4vW4jKdpmbtPf9lQS1bPumm1oYNFpkdxf0yE8OHj/", - "H+xvf37/l/d/ff9f7//ytz+//+/3f33/n91D+eMHs37LOmC5yIo8OU7ehT9vfIJVqasLUsID3JMzPHMX", - "vMqFrhsWKLyQqE+Nf3Nq59M3OrWUMO7tH0w8yG4j6vT7r/DP0ibHaERzwwsUb7I33kMDEwVfgL3Q5uJa", - "5KAxFPpvklGiK1dWjo5E4K0DRd3GZFL6+EMUXNCqPkmEpCGqo+NWoKjGYeNjeiUZ2FZXjnd06pqu2K6T", - "Hs2ZHgonEv474rqrSVgv7Zw53l6Phno6zGI0VMVsozNY8gHdqKbv1DSKsPxu+1KRLlPoUMXyCKThpe9H", - "RuJQ84z5Y0/lWLpmPDT40Uapk0kn5+SCXlWz2f5DJvUiuCM/cyTcFzYcE4QT+o2WTqdj06fhhYKxFCoc", - "Uqscc15gqyVHiFlz2Lj0p4KYydZR0SOesBfXYFboGywrDVwLXVm5pr3USJv+aCzPlHoRS3wXDInqDEUg", - "NgzJUvqyL5xRItGeFR4hcCMFnYwM+zo9Xdh1HCnWHiXpUBttW+fvNzTBIDPg4o9+YzNrM6gQpl4fKoqi", - "08d6vZUfZ2KhXnwoJ+q+1sX2c5iPvu1OT27LbgdU3bJrxx08XXK1gEj71yvRResoPqh5GU0BOsB2Iirf", - "RtVHoOUOCvpO1zpuHBVdfMWvfEfUSoASkw/focSytXI5FWkObFit53P0BBHfSsbie5xnSDVtb+UJuOBV", - "rKB+acGg7NHdogujxezk2YiV3NqVNnn9iKyDhusYd/VS0zF79DOeX74hz63IWsezdK5MbpBGoeaazkaV", - "45lrjyObY0t2DhyNrzIyvGmPp9N5nZ4JPR2eQv1AUy/PuSlYEVp6T05PklEiRQah6gl4vjr99vpgAH+1", - "Wk0WqsJsbRresdNFKccHk9kE1GTpCjoeEk72qA3oks7pabI3mU1muFqXoHgpMLXzX1Hd7iUz5aWYZpsn", - "Jgtydqih/rsTzOq+Atc/WkH9o3rJg9qfzWqWgvLv87KUoV0zfWMJNOnyXZoePcrxkutzXGE2KJu6jfSv", - "Kgpu1kQxtWW6YJp5rs5oguOYF/3k07PkdQ/GlyovtVDOB71FGFAaAGzk0AC9GRFvMYv19q5thKdUKNAh", - "b/Aif9T5+qPxsX+iPeSfn3jRoQRJug4FM/Obe5TwLQStuGW2yjKw80rKNaN5Sz8cGdKha5FXXNKI5mRj", - "6PWjUEfnMxH6/ANWH7/01Y2YzThTsPJDMagym5rRGR3pap5Xk77mfVOP1tGkKARF7KvWtIm822y2md+5", - "R2EOh4UirGsWtQNDEXuVg6EiP2/jGz79matbWNeiatj/pp3k7vHv3RudXoj8ZisLn4PLlmSh3ZGdn94l", - "AncVOqDBqxOwgSGNOny868zy9d/H6HxE7IvD79w/YDylmVcvux30ll5SeYhLBVJes72TVm7T2T81gz33", - "xorN8aRfHVwaDavPwTbiy+3h5akUvr+ZccUqG878nKb+NP0lLBZtFUdXyFt0odHZsJVyoakJkwTjVTtI", - "EA099chBGDi4n/gTKcsijG5L65r6TxqKBsMXu+jCJ4w5lYK3JWQOcgZhTVeFavJD4FnV8qy1LnzxOvJS", - "m8C0b9pNjbJiocZ6Pr8li8Eycz4fmuvhMNv//BgZyhXv0nuFyk+v0Rm3PPuOm6tuhcItqwuhO7j9lMsw", - "YkYa5k1cBgdSJwZXys/aw/oLA2yh6Q6SBz+Ji0TdIRF1r0YdUGw356bX+SltedgB+Icw5p118EnllqAc", - "NQRD2xG1oT77WDVjyB9ZIQ3wfI2rEB4NA/VaoaIV+FBdXei0RuN9R2TJ31szPKUs889Z29a5GW1zZmz7", - "G5+3Sn24elBKsmrnIw3QBZ71FibE9WCcdZpgUecVaZjdqyPrIoqw9/smNNI+d/Bn/1xxL/jzIDdiwoSd", - "++m0euaGY5KKDkNCTvk+HYQEX9IezPR0xQ/lCNVwpfYvYMZSZ1x618al/dj+7Bp6u6nsQFVduFy+Jbxm", - "S8grCec0GXp/dXX3qntEsP6Se7ehsM1Rfa/Dfdb+1TRfX9Q3V25GyeHs4OO1nnqjrhHiT8HUvY1noAQ5", - "zcPZ48hMNimgsExpV0c6OjEkdRoxq+vH/low9K7o0Nb9KTlTekVb3T/4tKGltiKukEqdOi6UT7s9dTRS", - "6W/SLbS/3ay097NkbR9osS8IOm/gd7hxlyl5nbJBwU2k7dSxkOk7f0YT2idxW+mcte7SQQkAf3sL5eOH", - "i85OttliyIeEIhLrHsYHR4vzJdSwVt61ZlDWETVqIufh7NdH5OA1umpEQvN24vqwvc104f+jhKWX7TE8", - "nUO7dSky3ybpnpqXRi8MWDsKd4DCpW7D5lzIysCdsaWOKBZU3uuGIbtr6OjFMCMiM6ELH9N6HHRKA923", - "xJP+yP49nQX0kUQE0puZbDI+XbnJr9PiGpfXs3ATrNvZ76rz/apaQwmXVMj4n3mwIRIc3j8B5z5dXuF/", - "xF4f+tRiwl5aYJe2J5vuwOQlCoIm+ZlnpZ911grs5HNqQj2lKwude+xUI9p1IYW6CtOVpEGBA3Sa4zCT", - "bJiC8Y9LyZb8Gug3O2hUkZxZGOxLYe6v9HEpm1/+aMNUa83E1A1rPgsEcWa72u6J6d1k4QZ43Jq7g6m7", - "2nRXpPdq37Hh6F1N/ZN2am6ZDY7RW6VBXigk5DjkvQnhUe3xSSWAhWFa2uLnZSt+9pzxWp+7PAg3GsL9", - "c22cDRZPkuKm2didmv4EE2FEk/kzhm4J3wfY1gRhlJqOFoiK1t/QrzE4IWVLQsc8PLzpu3qw/mb6zn8j", - "foEbMg5kjt1mJzRnqw08DYq4kSnufG3C/1jSMK2sl96aVw5mWYY/EfULbN77aC4NRLDWHNgFa3uL5vW9", - "W91gtjrW9ug3LD4/C+oOYLYz4NHbAHTBbGgst3nuRiP/fyvjKFZpBI9S59jhnkW4AJjDHAxrrhhQfPbc", - "8JH+VbI/e/QqaXs+fnzU18RKrlmKeYKrDNYv/meC2u3ZJnujuZDmTsdA4FRNc2k1wbC6AK2AgbQeTjtC", - "GyPTa4tn4BJ47s/SAgv/fUxoxk+5Gj/DfY5fegBJhIed2+AxHmojFkJx6XEi/Ak7mYcZXam7M73N3Rfh", - "mllbocLdFdF12X7strkPxxXjwq/IIa3oAuwOe3sRCBs/D4Qlt6nlzrW2zhy4sXUGeNH3EE05nwqF9j0s", - "6If5POGwGxfmfmWl7dVrUGfvzx7dtTyoY08RO+fyh3tHUQgmvI5FQMFdtmQpuBUEZQ/s7Ey71CMwYQ6A", - "CPCX1rQZ+J0mYa512Zc4DyLVGRlxuHN8h9XWFthaTlC80ugMrBdECvhigz9d9+yO0onLrSZ0zFBmlzT9", - "Rd6ly46wk88lAvnIEBps2+MO+177DgV3w4fePufaZCKVa5ZJbamX8fX5+SnLtFLgf/2CHFjdxgmOdy6U", - "sEuwPXkBg7c8c8zyAkIa6bSf/8dXcl1hhkcv2MkrVUv1C39XnKwp6EIKMQmwVOfrraG025dBFG1pMWRL", - "aPTgZwqoNAQ7TToHU4OfLeyPIQ3G6oSzIOeT1p/5YZuh6/1Gp/W5qW/g/FyBEWBHnVG70cbk0qQ332Uj", - "QJ+cnvSH/brHZrooKhVucKBLH86KNuBD/ykS64l/T05PRh6RV7lW+GFDvsWCf7/RaVPI2g78IK+b1zf/", - "FwAA//9HZfvdYFcAAA==", + "H4sIAAAAAAAC/+Q87W4cN5KvQvQekA1uvvRhy9av09pxIiOJhUjeHBAbEru7eoZSN9kh2RpPDAH7EPcm", + "dwvcj9tf9wLeNzqwiv01zZHGieR49/zDGHWTxWJ9V7HY76NEFaWSIK2JDt9HJllAwfHnkTFiLiE94+bK", + "/Z2CSbQorVAyOuy9ZcIwzqz7xQ0T1v2tIQFxDSmLV8wugP2o9BXoSTSKSq1K0FYArpKoouAyxd/CQoE/", + "/kVDFh1Gf5i2yE09ZtNnNCG6GUV2VUJ0GHGt+cr9faliN9s/NlYLOffPz0stlBZ21RkgpIU56HoEPQ1M", + "l7wIv7gdprHcVndux9HvlEa6HXFztRmRqhKpe5EpXXAbHdKD0frAm1Gk4edKaEijw5/qQY44fi8Nbp0t", + "rFGpQ5IuVqOWX2+bdVV8CYl1CB5dc5HzOIeXKj4Fax06A8k5FXKeAzP0nqmMcfZSxcxBMwEBWSiR0M8+", + "nB8XINlcXIMcsVwUwqKcXfNcpO7/Cgyzyj0zwDyQCXsl8xWrjMORLYVdMCIaLu7WbkRwQPx1YUsh41Vu", + "h3idLYD5l4QHMwu1lB4ZVhnQbOlwT8GCLoTE9RfC1CSZEPgOzPASzZOpVSq3ovQLCdku5ORRZzwBBAqp", + "sG7rBNHjn/HcwGhIXLsA7ZDmea6WzE1dR5TxzLoxC2CXKmYLblgMIJmp4kJYC+mE/aiqPGWiKPMVSyEH", + "mpbnDN4JQwC5uTIsU5pAX6p4xLhMnQFRRSlyN0bYyRvZCnqsVA5c4o6ueT6kz8nKLpRk8K7UYIxQSPwY", + "mBtdcQupo5HSKW2w5gPgTvqsa/BqeDMaisYVrIY4HKcgrcgEaA+kEfkRKypjHT6VFD9XJIieaZdeEYLr", + "OMXgeh7QhSO5YvDOas64nleFszC1vMXlauImmsmpKuCEdGv1xy9Z4thQGUjdyEQDt0Bb9fq36uDQqnhr", + "WT5ChERRQCq4hXzFNDhQjONWU8iEFG7CyBkCXN4tOUKaqMp6jLi2Iqlyrhs+bJAHU8W1+bzN6gYM1amf", + "2aj6R0M489OvhRHrSmZ1dRuBnOL2VcvLw+tjMpCOWLVaafbHXFwB4+xPOUgnxDxNx0p+OWGnYB24C2TI", + "BZkZ8sdcki2QPG/WsAtu3dJVnsovUCAbSwUyRQNiwoReczFOAfygLd3CacunNe9QxWP3hsSBFKLmOXtW", + "aQ3S5iumnB3nNVzUsI4lNxN28c3R6TdfPT9/cfztV+cnR2ffXFCUkgoNiVV6xUpuF+xf2cWbaPoH/Pcm", + "umC8LB1JU9o2yKpw+8tEDudufDSKUqHrn/jYe9QFNwtIz9uRbwMKvElohgbeU6Cz+47VIPfFDTt+Xusz", + "btsJjReJCfteMQnG2TpjdZXYSoNhf0T3ZUYsFYlbimsB5kvGNTBTlaXSdn3rHvmRi2z2dt2mc8VtNEJZ", + "uHOT4d3V3r5dk6JEYdh3XPI5aHIBwqLq88IZ6EBokPMY8o8L2Twxtw83QyHNIBpYUwcvEoReZ827dMNR", + "K2DcvxXG1sKA0r2ZbkMa1WHcr9vxWc8ibthuu0Rog3W8PtiWf8E0OC+NLoszQ8GhjzLREr2DpLJwVx6x", + "OUhvBKjzukYvzLjOlNCOvtJaaQdsPZNJoRed1xozTA0KMIbPQ/iuIYQw2/EhbF7kvACZqD+DNj5Y3JIy", + "1+2M27GoB3q9CmHxklIvnuevsujwp9sl7LSOD92sm9GAkBiLhCTGvcBoThRgLC9KZ49qcqfcwti9CYVO", + "IgDu9evj57WbeYnZ0R2J1bY5nTMVTUpXlek972aNO4hpTbN2vQbZtzdviUHfgeUptxwZlaYYdvH8pEf7", + "wY7X4kwdC6u5XrHCA/Nu10zYd0qj4pY5vOv6nIRL57UK5eJ/tFiV03J2wSfxJLlgUlmiQx0mXwGGnvCO", + "O1heoFHQDqPTUgsL7IUW84XzQi5GmUDBRe6wXsUa5L/F3gUqPa9HkA5EpziAndr//Z9ryDuGrSfIpx0f", + "EaYTRXPBuY2A1A6UJ1ZcY+bMZeIoQEl0mYP1vyURSyg5zrigEc2PkrsQPRpFP1dQ4Q+uk4W47vwk/0zg", + "x04y0O17IL0H+JugVI5E4+7i0ShackzyxpnSYxfJmKCD977mmZKZmFea26DZMQtecPmVdJ4kDWbvFPwu", + "gJ3iUOZWZFZzaTLQ7OjkGCO22htN7g5D+0uGzNQPMBfGgoaU/MgQbZ6mLmcM6kLOjT1HfvaLPp24QyRX", + "mz1Rzq3T73BgojK75HpD1LKV2aEttZaniRLOmwJOPwq4s8bxmwpODS1GDVG7haeaGKMooagesYzWqdyh", + "zIYdhfh8CkmlhV1tcNVb+9/bHC9J7bMFJFeqCtSBXC6mMpRqQ7UmuwCh2ek3R7uPHrPETTRVMWJG/IKh", + "e7yyYCjyTcE4FFiuErKNPh1M/GptGrNmKUlrnQPGJOQwajPsyVw5Ei54dBjtPYpn+093kt2DeLa3t5fu", + "ZPH+oyyZHTx5ynd2Ez57HO+kj/dn6e6jx08PnsziJ7ODFB7N9tOD2e5TmDlA4heIDnf2d/fRg9NquZrP", + "XabWWerxXnywmzzei5/u7+5n6c5e/HTvYJbFj2ezx09nT2bJHt95dLBzkGR7PN3f33289yjeeXKQPOZP", + "nj6aHTxtl9o9QE+2Xh4kipwgAoNCkMvxlgvQVNvxlsvnvL2iRw1nwo59/TrnznDXZRRvoxoGYPbIDUu8", + "FYSUKdldZMKOJVO5S899/GTqYMPDwnWX3LBLl9i5F2+a7bDj52+iEYsrS6wXpobi8nnvUjlhgcWAC+8j", + "xyav5lOTgISx074p1ZjGx88veql8q/ReZLbMDgj3FyKH0xKSO3MDAj7qs+lubfoBzMbaqsZ3VBZZ40qo", + "evwrxMMHU+uCcYZ/EulTkWXgrBazCy7ZcsEtsrLxtiMnHF2gS5HnDKSptGOcr/y1aszc1pCd9yJ8IVav", + "5xbbsaRh9dDAlZCITHgLhfxAF+5tlUfaWKX5HIasKYMs+Va18ByULsQa42BkvuABDPumtgszCAPtzDoQ", + "R4C+jQ7kdMNIpLZbo6jcjsA/Crtow8etSD1iy4VIFixBcxZvIP2IKe2i7BFLoQSZ4qmLxOoKueN/ct5s", + "Gz912OFjqDu52iZZt7N3kBVU8kqqpcR8LVc8pRjeMawXubb7J2A/EDZY4P+BTM2vDjww0OjRbmMs8UBB", + "wycJED6Be9vM/D6/TKmkgbBXI25lWhWMM92ZVruUUZeVJHW10a/VHfS1izteICgs7nMNDAXNeRI/zD2D", + "d0lepZDSgpa8KmL3KWWgVcxGHx5GLLoLNep2z7LSMd+/VWrohLxvONZU3PP/Y33ufRnCW4xet7YXrPy3", + "GUl7UOzEsy5krklg0alaPVwdyL/Y+/Af7O9/+fDXD3/78F8f/vr3v3z47w9/+/Cf3T6Jw0ez/imCX+U8", + "KdLoMHrv/7zBmLeSV+ckhHtuT1bzxJ7zKhWqriE55vncaapx5tRk00sVG4rhd3b3JgiyWxs8+f5r92dp", + "okOnRJnmhWNvtDPecQomCj4Hc670+bVIQTlXiE+iUaQqW1aWTqngnQVJBeBoUqL/IQzOaVQfJVqkQaoj", + "40Y4Vo39xsc0JRroVpePdxRPm0Llts03zTGrY07A/XfYdVfdth7aOQa+PYz2JQ7fHtNgFdKNTq/PRxQI", + "m1JgU7szKrNtqTBQ+PNFw1Ac4XB4jSXigB9q3jE8iZaWxSvG/ZmL01EqLlMzA5mgN9VstvuY5WruzRG2", + "gQn7hfEnN75pYq3K1imi9XF4JWGcC+n7BmTqYl7A/OoLw5Lm/HeBB7Uukq29Ii48Ya+uQS+dbTCs1HAt", + "VGXyFe2lXrQpWYfizFzNQ4HvnDmkOn0qbrURJXUuE/fHxg5pJAUuCFzngg6rhqW2nixs2yEWqlgTd6iy", + "uakY+xvqkpBosOFXv7G+uO5UaKVeaTC4RKe0+HYjPU7FXL76WErUpcbzzUdj977tTpl0w24HWN2ya8st", + "PFtwOYdARR6F6Lw1FB9VTw6GAB1gWyGVbsLqHnC5A4O+0TWWa0tJF1/yKyxSmxygdMEHFo1d2lrZlJI0", + "C8aPVlnmLEHAtpKyYNn51GFN21siAue8CiXUrw1ox3tnbp0Jo8Hs+PmIldyYpdJp/Yq0g/odGbf1UN1R", + "e2dnkF54RsKNSFrDs7C2jG4cjkJmio6rpeWJbU+Im5NkdgbcKV+lcz/THE6nWR2eCTUdHgz+QI1IL7gu", + "WOErXkcnx9EoykUCPuvx63x98u313gD+crmczGXlorWpn2Om8zIf701mE5CThS3oxE7YvIetXy7qHGhH", + "O5PZZOZGqxIkL4UL7fAR5e3ImSkvxTRZP8Sak7FzEorPjl1U9zXY/mmXkz/KlxDU7mxWkxQkzudlmfty", + "zfTSEGiS5bskPXi6hpzrU1y6aDBv8jaSv6oouF4RxlSW6YJpWuw63SKWu7joJwzPorc9GF/JtFRCWnR6", + "c98zNgDY8KEBejMi2rooFvVdmQBNKVGgc3dvRf6k0tW90bHfZDCkHzYhKZ+CRF2D4iLzmwfk8C0ILblh", + "pkoSMFmV5ytGLbDYr+rDoWuRVjynrtnJWh/yvWBHR2YB/PAFq0/E+uJGxGacSVhin5ITmXXJ6HTzdCUP", + "xaQveS/rbkdq3gUviH3Rmjaed5PONi1VD8jMYf9WgHTNoLaHK6Cv+aDPC1ugsODTb4O7hXTtUg35L9vm", + "+h793l+q+FykNxtJ+AJssiAN7XZR/fQ+Em5XvgLqrToBGyjSqEPHu46R3/4+Socesc8O3Dm+YDymNmTk", + "3RZyS5Nk6v1S4TCvyd4JKzfJ7J+bXqsHI8V6x9ivdi6NhNVHk2v+5Xb38iwXWN9MuGSV8cewVlF9mv4S", + "xiVtFXemkLfL+UJnQ1aKhabaN3eMl21vR9D11F0gvgfkYfxPIC0LELpNrWvsP6krGvTDbCMLn9DnVBLe", + "lZBYSBn4MV0RqtH3jmdZ87OWOv/gbWBSG8C0M826RBkxl2OVZbdEMS7NzLKhuu4Po/3Pj5A+XUGT3ktU", + "fnrrjHFLs++4vupmKNywOhG6g9rPeO67/kjCUMVzb0DqwOBK4vUHWH2hgc0VXQtD8JMwS+QdHJEPqtR+", + "ic3q3NQ6P6UuDysA/xDKvLUMHlV2AdJSQdCXHZ001Gcfy6Yz/J4FUgNPV26Ug0f9Wb1SqGgZPhRX6yut", + "QX/fYVn0e0sGYsoSfN+2sbj9bDBmbPOMz1ukPl48KCRZti2rGuhO1WoDEcJyME46RbCg8QoUzB7UkHUX", + "CpD3+8Y10j63sGf/XH7P23PPNyJC3QhW99xwF6Q6g5FDSvE+HYR4W9IezPRkBZtyhGyoUtsX0ONcJTxH", + "08Zzc9/27Bp6u6nMQFStv++/wb0mC0irHM6oWffh8uru1wcCjMXvDnQLCpsM1ffKXzHu3xbE/KK+THQz", + "ivZne/dXeup1HweQPwFd1zaegxRkNPdnTwNt8iSAwjCpbO3p6MSQxGnEjKpf401t6N2aoq3jKTmTaklb", + "3d37tK6l1iIuHZYqtlxIDLsRO+pyxcuNc4UXzqVCO0va9pEa+4qg8wZ+hxp3qRLKlPECrgNlp46GTN/j", + "GY0vn4R1pXPWuk0FxQP87SWU+3cXnZ1s0kUfDwlJKNY1jI/2FmcLqGEt0bQmUNYeNagiZ/7sFz2ytxpd", + "MSKmoZ7YPmzUmS78fxS39Lo9hqdzaLsqRYJlku6peanVXIMxI38ty9+z1yzjIq803Olbao9iQKa9apgj", + "dw3dWTEXEZGa0B2cad0OOqVe6lv8Sf8WxQOdBfQXCTCk1zPZRHy+pfzT5XDBLvgAuvUIFOO6Xb1zcNDV", + "loeV5AYTnlOehB/2MN7R7D88AmcYjS/df8Q99KxyPmGvDbALs0bRtrHywvGZ2ucZkhJbqZUEM/mcalzP", + "6JJK58sFlIKaVZELeeWbN0lAPQXosMjSnQJPFOdeeZ6zBb8G+koLdUKSrfR9gzFkeImT53nzrZfWC7bG", + "goi6ZixOPUKcma4yITK9u0tcAw8bi27f67Ymo8vSBzUfod7rbS3J72BEgq3HIXyr2PPLMclRHNJeA/Ko", + "digkEsB8ry5t8fPSFWxtb+8FdWngL0z4Lw4obY3XeOIU183G7pT0Ixdnu2USPMLoVgj6ANuUw3dq08kF", + "YdHaG/r+hhV53qLQUQ+EN31f9+3fTN/jE/EL3JByOOKYTXpCbbxKwzMviGuB6Na3MvDzWMOotR56a9g6", + "aJUZfhTsF1i/VtLcSQisWlNgm1XbSzpvH1zrBq3boapKvx7y+WlQt7+zbTEPXjage2ZDZbnNcjcS+f9b", + "GEehRMZblDqE99c4/JXPFDLQrLnBQP4ZqYGe/k20O3vyJmpLStidiim3zFcsdnGCrbRLj/DDUO32TBO9", + "UdtJc2VkwHBK1nluFMEwqgAlgUFuEE7boRtCE6UFCbgAnuJRnSfhv49pmfEzLsfP3T7HrxFAFKBh5/5/", + "iIZKi7mQPMc1HfwJO858C3Cuui3DzdUaYZtWXiH91RjRNdnY1dtct+OScYEjUogruvK8xd5eecTGLzxi", + "0W1iuXUqrxILdmysBl70LURTLYiFdPo9rBcM43law6zdx/uViTyK1yCN3509uWu4F8eeIHaO/fd3DoIQ", + "tJ/ukoCC22TBYrBL8MLuydlppqk7bHybASGAd+KUHtidJmCuZRlTnEeBL7uQEvtb5ndoba2BreZ4wSu1", + "SsAgI2JwE5v141VP7yicuNioQofM8eyCmsvIunTJ4XfyuXgg9Ay+frfZ77DvFRZAuB2+RP3MlE5EnK9Y", + "kitDpZJvzs5OWKKkBPzeCRmwukrkDW8mpDALMD1+AYN3PLHM8AJ8GGkVXi9wU1JVuQiPJpjJG1lz9Qu8", + "oE3a5GUhhhAHWKzS1UZX2i37uCXa1GJIFl9Hcr/JoVKP7TTqnHsNPlTZ73IadO0JayDPJq09w16eoel9", + "qeL6WBbrQz9XoAWYUaeTb7TWGDXptY+ZANCjk+N+L2H3VE4VRSX9BRFn0oetqA14X94K+Hqi39HJ8QgX", + "QpFrme83hCUW9/eliptE1nTge37dvL35vwAAAP//0IX4BlJZAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/pkg/api/openapi_types.gen.go b/pkg/api/openapi_types.gen.go index 498d5939..34021a66 100644 --- a/pkg/api/openapi_types.gen.go +++ b/pkg/api/openapi_types.gen.go @@ -250,11 +250,17 @@ type SecurityError struct { // Set of files with their SHA256 checksum, size in bytes, and desired location in the checkout directory. type ShamanCheckout struct { - // Path where the Manager should create this checkout, It is relative to the Shaman checkout path as configured on the Manager. In older versions of the Shaman this was just the "checkout ID", but in this version it can be a path like `project-slug/scene-name/unique-ID`. + // Path where the Manager should create this checkout. It is relative to the Shaman checkout path as configured on the Manager. In older versions of the Shaman this was just the "checkout ID", but in this version it can be a path like `project-slug/scene-name/unique-ID`. CheckoutPath string `json:"checkoutPath"` Files []ShamanFileSpec `json:"files"` } +// The result of a Shaman checkout. +type ShamanCheckoutResult struct { + // Path where the Manager created this checkout. This can be different than what was requested, as the Manager will ensure a unique directory. The path is relative to the Shaman checkout path as configured on the Manager. + CheckoutPath string `json:"checkoutPath"` +} + // Specification of a file in the Shaman storage. type ShamanFileSpec struct { // Location of the file in the checkout diff --git a/pkg/shaman/checkout/checkout.go b/pkg/shaman/checkout/checkout.go index bdce558b..2144ef0c 100644 --- a/pkg/shaman/checkout/checkout.go +++ b/pkg/shaman/checkout/checkout.go @@ -20,7 +20,7 @@ var ( validCheckoutRegexp = regexp.MustCompile(`^[^/?*:;{}\\][^?*:;{}\\]*$`) ) -func (m *Manager) Checkout(ctx context.Context, checkout api.ShamanCheckout) error { +func (m *Manager) Checkout(ctx context.Context, checkout api.ShamanCheckout) (string, error) { logger := (*zerolog.Ctx(ctx)).With(). Str("checkoutPath", checkout.CheckoutPath).Logger() logger.Debug().Msg("shaman: user requested checkout creation") @@ -28,7 +28,7 @@ func (m *Manager) Checkout(ctx context.Context, checkout api.ShamanCheckout) err // Actually create the checkout. resolvedCheckoutInfo, err := m.PrepareCheckout(checkout.CheckoutPath) if err != nil { - return err + return "", err } // The checkout directory was created, so if anything fails now, it should be erased. @@ -46,17 +46,17 @@ func (m *Manager) Checkout(ctx context.Context, checkout api.ShamanCheckout) err blobPath, status := m.fileStore.ResolveFile(fileSpec.Sha, int64(fileSpec.Size), filestore.ResolveStoredOnly) if status != filestore.StatusStored { // Caller should upload this file before we can create the checkout. - return ErrMissingFiles + return "", ErrMissingFiles } if err := m.SymlinkToCheckout(blobPath, resolvedCheckoutInfo.absolutePath, fileSpec.Path); err != nil { - return fmt.Errorf("symlinking %q to checkout: %w", fileSpec.Path, err) + return "", fmt.Errorf("symlinking %q to checkout: %w", fileSpec.Path, err) } } checkoutOK = true // Prevent the checkout directory from being erased again. logger.Info().Msg("shaman: checkout created") - return nil + return resolvedCheckoutInfo.RelativePath, nil } func isValidCheckoutPath(checkoutPath string) bool { diff --git a/pkg/shaman/checkout/checkout_test.go b/pkg/shaman/checkout/checkout_test.go index c64c4bed..37c4dce3 100644 --- a/pkg/shaman/checkout/checkout_test.go +++ b/pkg/shaman/checkout/checkout_test.go @@ -28,13 +28,13 @@ func TestCheckout(t *testing.T) { }, } - err := manager.Checkout(ctx, checkout) + actualCheckoutPath, err := manager.Checkout(ctx, checkout) if err != nil { t.Fatalf("fatal error: %v", err) } // Check the symlinks of the checkout - coPath := path.Join(manager.checkoutBasePath, checkout.CheckoutPath) + coPath := path.Join(manager.checkoutBasePath, actualCheckoutPath) assert.FileExists(t, path.Join(coPath, "subdir", "replacer.py")) assert.FileExists(t, path.Join(coPath, "feed.py")) assert.FileExists(t, path.Join(coPath, "httpstuff.py")) diff --git a/pkg/shaman/checkout/manager.go b/pkg/shaman/checkout/manager.go index b8215073..1dabdbae 100644 --- a/pkg/shaman/checkout/manager.go +++ b/pkg/shaman/checkout/manager.go @@ -25,6 +25,7 @@ package checkout import ( "errors" "fmt" + "math/rand" "os" "path" "path/filepath" @@ -43,7 +44,9 @@ type Manager struct { checkoutBasePath string fileStore *filestore.Store - wg sync.WaitGroup + wg *sync.WaitGroup + + checkoutUniquenessMutex *sync.Mutex } // ResolvedCheckoutInfo contains the result of validating the Checkout ID and parsing it into a final path. @@ -78,7 +81,7 @@ func NewManager(conf config.Config, fileStore *filestore.Store) *Manager { logger.Error().Err(err).Msg("unable to create checkout directory") } - return &Manager{conf.CheckoutPath, fileStore, sync.WaitGroup{}} + return &Manager{conf.CheckoutPath, fileStore, new(sync.WaitGroup), new(sync.Mutex)} } // Close waits for still-running touch() calls to finish, then returns. @@ -101,37 +104,54 @@ func (m *Manager) pathForCheckout(requestedCheckoutPath string) (ResolvedCheckou // PrepareCheckout creates the root directory for a specific checkout. // Returns the path relative to the checkout root directory. func (m *Manager) PrepareCheckout(checkoutPath string) (ResolvedCheckoutInfo, error) { - checkoutPaths, err := m.pathForCheckout(checkoutPath) - if err != nil { - return ResolvedCheckoutInfo{}, err - } + // This function checks the filesystem and tries to ensure uniqueness, so it's + // important that it doesn't run simultaneously in parallel threads. + m.checkoutUniquenessMutex.Lock() + defer m.checkoutUniquenessMutex.Unlock() - logger := log.With(). - Str("absolutePath", checkoutPaths.absolutePath). - Str("checkoutPath", checkoutPath). - Logger() + var lastErr error + // Just try 10 different random tokens. If that still doesn't work, fail. + for try := 0; try < 10; try++ { + randomisedPath := fmt.Sprintf("%s-%s", checkoutPath, randomisedToken()) - if stat, err := os.Stat(checkoutPaths.absolutePath); !os.IsNotExist(err) { - if err == nil { - if stat.IsDir() { - logger.Debug().Msg("shaman: checkout path exists") - } else { - logger.Error().Msg("shaman: checkout path exists but is not a directory") - } - // No error stat'ing this path, indicating it's an existing checkout. - return ResolvedCheckoutInfo{}, ErrCheckoutAlreadyExists + checkoutPaths, err := m.pathForCheckout(randomisedPath) + if err != nil { + return ResolvedCheckoutInfo{}, err } - // If it's any other error, it's really a problem on our side. - logger.Error().Err(err).Msg("shaman: unable to stat checkout directory") - return ResolvedCheckoutInfo{}, err + + logger := log.With(). + Str("absolutePath", checkoutPaths.absolutePath). + Str("checkoutPath", checkoutPath). + Logger() + + if stat, err := os.Stat(checkoutPaths.absolutePath); !os.IsNotExist(err) { + if err == nil { + // No error stat'ing this path, indicating it's an existing checkout. + // Just retry another random token. + lastErr = ErrCheckoutAlreadyExists + if stat.IsDir() { + logger.Debug().Msg("shaman: checkout path exists") + } else { + logger.Warn().Msg("shaman: checkout path exists but is not a directory") + } + continue + } + // If it's any other error, it's really a problem on our side. Don't retry. + logger.Error().Err(err).Msg("shaman: unable to stat checkout directory") + return ResolvedCheckoutInfo{}, err + } + + if err := os.MkdirAll(checkoutPaths.absolutePath, 0777); err != nil { + lastErr = err + logger.Warn().Err(err).Msg("shaman: unable to create checkout directory") + continue + } + + logger.Info().Str("relPath", checkoutPaths.RelativePath).Msg("shaman: created checkout directory") + return checkoutPaths, nil } - if err := os.MkdirAll(checkoutPaths.absolutePath, 0777); err != nil { - logger.Error().Err(err).Msg("shaman: unable to create checkout directory") - } - - logger.Info().Str("relPath", checkoutPaths.RelativePath).Msg("shaman: created checkout directory") - return checkoutPaths, nil + return ResolvedCheckoutInfo{}, lastErr } // EraseCheckout removes the checkout directory structure identified by the ID. @@ -231,3 +251,17 @@ func touchFile(blobPath string) error { logger.Debug().Msg("done touching") return nil } + +// randomisedToken generates a random 4-character string. +// It is intended to add to a checkout path, to create some randomness and thus +// a higher chance of the path not yet existing. +func randomisedToken() string { + var runes = []rune("abcdefghijklmnopqrstuvwxyz0123456789") + + n := 4 + s := make([]rune, n) + for i := range s { + s[i] = runes[rand.Intn(len(runes))] + } + return string(s) +} diff --git a/pkg/shaman/server.go b/pkg/shaman/server.go index b0782b27..d7edac36 100644 --- a/pkg/shaman/server.go +++ b/pkg/shaman/server.go @@ -112,7 +112,7 @@ func (s *Server) IsEnabled() bool { // Checkout creates a directory, and symlinks the required files into it. The // files must all have been uploaded to Shaman before calling this. -func (s *Server) Checkout(ctx context.Context, checkout api.ShamanCheckout) error { +func (s *Server) Checkout(ctx context.Context, checkout api.ShamanCheckout) (string, error) { return s.checkoutMan.Checkout(ctx, checkout) }