diff --git a/internal/manager/api_impl/api_impl.go b/internal/manager/api_impl/api_impl.go index d06b9e65..d076d41c 100644 --- a/internal/manager/api_impl/api_impl.go +++ b/internal/manager/api_impl/api_impl.go @@ -86,10 +86,10 @@ type Shaman interface { // Requirements checks a Shaman Requirements file, and returns the subset // containing the unknown files. - Requirements(ctx context.Context, requirements api.ShamanRequirements) (api.ShamanRequirements, error) + Requirements(ctx context.Context, requirements api.ShamanRequirementsRequest) (api.ShamanRequirementsResponse, error) // Check the status of a file on the Shaman server. - FileStoreCheck(ctx context.Context, checksum string, filesize int64) (api.ShamanFileStatusStatus, error) + FileStoreCheck(ctx context.Context, checksum string, filesize int64) (api.ShamanFileStatus, error) // Store a new file on the Shaman server. Note that the Shaman server can // return early when another client finishes uploading the exact same file, to diff --git a/internal/manager/api_impl/mocks/api_impl_mock.gen.go b/internal/manager/api_impl/mocks/api_impl_mock.gen.go index db40c763..325ff785 100644 --- a/internal/manager/api_impl/mocks/api_impl_mock.gen.go +++ b/internal/manager/api_impl/mocks/api_impl_mock.gen.go @@ -440,10 +440,10 @@ func (mr *MockShamanMockRecorder) FileStore(arg0, arg1, arg2, arg3, arg4, arg5 i } // FileStoreCheck mocks base method. -func (m *MockShaman) FileStoreCheck(arg0 context.Context, arg1 string, arg2 int64) (int, error) { +func (m *MockShaman) FileStoreCheck(arg0 context.Context, arg1 string, arg2 int64) (api.ShamanFileStatus, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "FileStoreCheck", arg0, arg1, arg2) - ret0, _ := ret[0].(int) + ret0, _ := ret[0].(api.ShamanFileStatus) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -455,10 +455,10 @@ func (mr *MockShamanMockRecorder) FileStoreCheck(arg0, arg1, arg2 interface{}) * } // Requirements mocks base method. -func (m *MockShaman) Requirements(arg0 context.Context, arg1 api.ShamanRequirements) (api.ShamanRequirements, error) { +func (m *MockShaman) Requirements(arg0 context.Context, arg1 api.ShamanRequirementsRequest) (api.ShamanRequirementsResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Requirements", arg0, arg1) - ret0, _ := ret[0].(api.ShamanRequirements) + ret0, _ := ret[0].(api.ShamanRequirementsResponse) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/internal/manager/api_impl/shaman.go b/internal/manager/api_impl/shaman.go index 00109bce..6a3e83ab 100644 --- a/internal/manager/api_impl/shaman.go +++ b/internal/manager/api_impl/shaman.go @@ -47,7 +47,7 @@ func (f *Flamenco) ShamanCheckoutRequirements(e echo.Context) error { return sendAPIError(e, http.StatusBadRequest, "invalid format") } - unknownFiles, err := f.shaman.Requirements(e.Request().Context(), api.ShamanRequirements(reqBody)) + unknownFiles, err := f.shaman.Requirements(e.Request().Context(), api.ShamanRequirementsRequest(reqBody)) if err != nil { logger.Warn().Err(err).Msg("Shaman: checking checkout requirements file") return sendAPIError(e, http.StatusInternalServerError, "unexpected error: %v", err) @@ -71,11 +71,11 @@ func (f *Flamenco) ShamanFileStoreCheck(e echo.Context, checksum string, filesiz // TODO: actually switch over the actual statuses, see the TODO in the Shaman interface. switch status { - case api.ShamanFileStatusStatusStored: + case api.ShamanFileStatusStored: return e.String(http.StatusOK, "") - case api.ShamanFileStatusStatusUploading: + case api.ShamanFileStatusUploading: return e.String(420 /* Enhance Your Calm */, "") - case api.ShamanFileStatusStatusUnknown: + case api.ShamanFileStatusUnknown: return e.String(http.StatusNotFound, "") } diff --git a/pkg/api/flamenco-manager.yaml b/pkg/api/flamenco-manager.yaml index 31698ca3..5b2bddd5 100644 --- a/pkg/api/flamenco-manager.yaml +++ b/pkg/api/flamenco-manager.yaml @@ -272,13 +272,13 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ShamanRequirements" + $ref: "#/components/schemas/ShamanRequirementsRequest" responses: "200": description: Subset of the posted requirements, indicating the unknown files. content: application/json: - schema: {$ref: "#/components/schemas/ShamanRequirements"} + schema: {$ref: "#/components/schemas/ShamanRequirementsResponse"} default: description: unexpected error content: @@ -307,6 +307,12 @@ paths: responses: "204": description: Checkout was created succesfully. + "424": + description: There were files missing. Use `shamanCheckoutRequirements` to figure out which ones. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' "409": description: Checkout already exists. content: @@ -344,7 +350,12 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ShamanFileStatus' + type: object + description: Status of a file in the Shaman storage. + properties: + "status": {$ref: "#/components/schemas/ShamanFileStatus"} + required: [status] + default: description: unexpected error content: @@ -677,57 +688,80 @@ components: properties: message: {type: string} - ShamanRequirements: + ShamanRequirementsRequest: type: object description: Set of files with their SHA256 checksum and size in bytes. properties: - "req": + "files": type: array - items: - type: object - properties: - "c": {type: string, description: "SHA256 checksum of the file"} - "s": {type: integer, description: "File size in bytes"} - required: [c, s] - required: [req] + items: {$ref: "#/components/schemas/ShamanFileSpec"} + required: [files] example: - req: - - c: 35b0491c27b0333d1fb45fc0789a12ca06b1d640d2569780b807de504d7029e0 - s: 1424 - - c: 63b72c63b9424fd13b9370fb60069080c3a15717cf3ad442635b187c6a895079 - s: 127 + files: + - sha: 35b0491c27b0333d1fb45fc0789a12ca06b1d640d2569780b807de504d7029e0 + size: 1424 + - sha: 63b72c63b9424fd13b9370fb60069080c3a15717cf3ad442635b187c6a895079 + size: 127 + + ShamanRequirementsResponse: + type: object + description: The files from a requirements request, with their status on the Shaman server. Files that are known to Shaman are excluded from the response. + properties: + "files": + type: array + items: {$ref: "#/components/schemas/ShamanFileSpecWithStatus"} + required: [files] + example: + files: + - sha: 35b0491c27b0333d1fb45fc0789a12ca06b1d640d2569780b807de504d7029e0 + size: 1424 + status: unknown + - sha: 63b72c63b9424fd13b9370fb60069080c3a15717cf3ad442635b187c6a895079 + size: 127 + status: uploading + + ShamanFileSpec: + type: object + description: Specification of a file in the Shaman storage. + properties: + "sha": {type: string, description: "SHA256 checksum of the file"} + "size": {type: integer, description: "File size in bytes"} + required: [sha, size] + + ShamanFileSpecWithStatus: + allOf: + - $ref: '#/components/schemas/ShamanFileSpec' + - properties: + "status": {$ref: "#/components/schemas/ShamanFileStatus"} + required: [status] + + ShamanFileSpecWithPath: + allOf: + - $ref: '#/components/schemas/ShamanFileSpec' + - properties: + "path": {type: string, description: Location of the file in the checkout} + required: [path] ShamanCheckout: type: object description: Set of files with their SHA256 checksum, size in bytes, and desired location in the checkout directory. properties: - "req": + "files": type: array - items: - type: object - properties: - "c": {type: string, description: "SHA256 checksum of the file"} - "s": {type: integer, description: "File size in bytes"} - "p": {type: string, description: "File checkout path"} - required: [c, s, p] - required: [req] + items: {$ref: "#/components/schemas/ShamanFileSpecWithPath"} + required: [files] example: - req: - - c: 35b0491c27b0333d1fb45fc0789a12ca06b1d640d2569780b807de504d7029e0 - s: 1424 - p: definition.go - - c: 63b72c63b9424fd13b9370fb60069080c3a15717cf3ad442635b187c6a895079 - s: 127 - p: logging.go + files: + - sha: 35b0491c27b0333d1fb45fc0789a12ca06b1d640d2569780b807de504d7029e0 + size: 1424 + path: definition.go + - sha: 63b72c63b9424fd13b9370fb60069080c3a15717cf3ad442635b187c6a895079 + size: 127 + path: logging.go ShamanFileStatus: - type: object - description: Status of a file in the Shaman storage. - properties: - "status": - type: string - enum: [unknown, uploading, stored] - required: [status] + type: string + enum: [unknown, uploading, stored] securitySchemes: worker_auth: diff --git a/pkg/api/openapi_client.gen.go b/pkg/api/openapi_client.gen.go index d59dc9d5..afee6945 100644 --- a/pkg/api/openapi_client.gen.go +++ b/pkg/api/openapi_client.gen.go @@ -1338,6 +1338,7 @@ type ShamanCheckoutResponse struct { Body []byte HTTPResponse *http.Response JSON409 *Error + JSON424 *Error JSONDefault *Error } @@ -1360,7 +1361,7 @@ func (r ShamanCheckoutResponse) StatusCode() int { type ShamanCheckoutRequirementsResponse struct { Body []byte HTTPResponse *http.Response - JSON200 *ShamanRequirements + JSON200 *ShamanRequirementsResponse JSONDefault *Error } @@ -1383,8 +1384,10 @@ func (r ShamanCheckoutRequirementsResponse) StatusCode() int { type ShamanFileStoreCheckResponse struct { Body []byte HTTPResponse *http.Response - JSON200 *ShamanFileStatus - JSONDefault *Error + JSON200 *struct { + Status ShamanFileStatus `json:"status"` + } + JSONDefault *Error } // Status returns HTTPResponse.Status @@ -1406,7 +1409,6 @@ func (r ShamanFileStoreCheckResponse) StatusCode() int { type ShamanFileStoreResponse struct { Body []byte HTTPResponse *http.Response - JSON409 *Error JSONDefault *Error } @@ -1966,6 +1968,13 @@ func ParseShamanCheckoutResponse(rsp *http.Response) (*ShamanCheckoutResponse, e } response.JSON409 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 424: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON424 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { @@ -1993,7 +2002,7 @@ func ParseShamanCheckoutRequirementsResponse(rsp *http.Response) (*ShamanCheckou switch { case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest ShamanRequirements + var dest ShamanRequirementsResponse if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } @@ -2026,7 +2035,9 @@ func ParseShamanFileStoreCheckResponse(rsp *http.Response) (*ShamanFileStoreChec switch { case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest ShamanFileStatus + var dest struct { + Status ShamanFileStatus `json:"status"` + } if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } @@ -2058,13 +2069,6 @@ func ParseShamanFileStoreResponse(rsp *http.Response) (*ShamanFileStoreResponse, } switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON409 = &dest - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { diff --git a/pkg/api/openapi_spec.gen.go b/pkg/api/openapi_spec.gen.go index 8c665578..33798a93 100644 --- a/pkg/api/openapi_spec.gen.go +++ b/pkg/api/openapi_spec.gen.go @@ -18,86 +18,90 @@ import ( // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+w86W4cR3qvUugN4DXSc/AQKfFXuJJlU7AtwqTWASyCqu7+ZqbE6qp2VTVHY4HAPkTe", - "JFkgP7K/8gLaNwq+OvqYriFHNilrk/iHMeyu47vv1vskl2UlBQijk6P3ic4XUFL781hrNhdQnFN9hX8X", - "oHPFKsOkSI56bwnThBKDv6gmzODfCnJg11CQbEXMAsiPUl2BGidpUilZgTIM7C25LEsqCvubGSjtj39S", - "MEuOkj9MWuAmHrLJU7chuUkTs6ogOUqoUnSFf7+VGe72j7VRTMz988tKMamYWXUWMGFgDiqscE8j2wUt", - "4y9uP1Mbauo70UH6nbmViBHVV5sBqWtW4IuZVCU1yZF7kK4vvEkTBT/XTEGRHP0UFiFxPC4NbB0U1qjU", - "IUkXqrTl10Vzr8zeQm4QwONryjjNOLyQ2RkYg+AMJOeMiTkHot17ImeEkhcyI3iajgjIQrLc/eyf8+MC", - "BJmzaxAp4axkxsrZNeWswP/XoImR+EwD8YeMyUvBV6TWCCNZMrMgjmj2cry7EcEB8deFrYAZrbkZwnW+", - "AOJfOjiIXsil8MCQWoMiS4S9AAOqZMLev2A6kGTsju+cGb+ieTIxUnLDKn8RE+1FKI9qRnOwh0LBDKLu", - "TvTwzyjXkA6JaxagEGjKuVwS3LoOKKEzg2sWQN7KjCyoJhmAILrOSmYMFGPyo6x5QVhZ8RUpgIPbxjmB", - "d0y7A6m+0mQmlTv6rcxSQkWBBkSWFeO4hpnxa9EKeiYlByosRteUD+lzujILKQi8qxRozaQlfgYEV9fU", - "QIE0kqpwCAY+gMWkz7oGroY36VA0rmA1hOGkAGHYjIHyhzQin5Ky1gbhqQX7uXaC6Jn21itC9B5UDKrm", - "EV04FisC74yihKp5XaKFCfKWVasxbtTjM1nCqdOt1R+/JDmyodZQ4MpcATXgUPX6t+rA0Kp4a1k+QoRY", - "WULBqAG+IgrwKEItqgXMmGC4IUVDYK/HK1NLE1kbDxFVhuU1p6rhwwZ50HUWzOdtVjdiqM78zkbVP/qE", - "c7/9mmm2rmRG1bcRCBW3r1peHl6dOAOJxApqpcgfObsCQsmfOAgUYloUIym+HJMzMHjcG8uQN87MOH9M", - "hbMFgvLmDrOgBq+ueSG+sALZWCoQhTUgOk7oNReDCuAXbekWzlo+rXmHOhvhGycOTiECz8nTWikQhq+I", - "RDtOw7lWwzqWXI/Jm2+Oz7756tnl85Nvv7o8PT7/5o2LUgqmIDdSrUhFzYL8M3nzOpn8wf73OnlDaFUh", - "SQuHNoi6RPxmjMMlrk/SpGAq/LSPvUddUL2A4rJdeRFR4E1CMzTwngId7DtWw7kvqsnJs6DPFm0UGi8S", - "Y/K9JAI02jptVJ2bWoEmf7TuS6ekYDleRRUD/SWhCoiuq0oqs466Bz7FyGZvF5HmkpoktbJwJ5Jx7IK3", - "b+90USLT5Dsq6ByUcwHMWNWnJRroSGjAaQb840I2T8ztw81YSDOIBtbUwYuEA69z5126gdSKGPdvmTZB", - "GKx0b6bbkEYhjPt1GJ/3LOIGdNsrYgiGeH2Aln9BFKCXti6LEu2CQx9lWkv0DvLawF15xOYgvRGgzusA", - "XpxxnS0xjL5SSio8bD2TKaAXnQeNGaYGJWhN5zF41wCyZ7brY9A857QEkcs/g9I+WNySMtftjtuhCAu9", - "XsWgeOFSL8r5y1ly9NPtEnYW4kPcdZMOCGljkZjE4AsbzbEStKFlhfYokLugBkb4JhY6schxr16dPAtu", - "5oXNju5IrLbN6dBUNCldXRX3jM0adyykgWbtfQ2wFzcXjkHfgaEFNdQyqihs2EX5aY/2A4zX4kyVMaOo", - "WpHSH+bdrh6T76SyiltxeNf1OTkV6LVKifG/tVg1ajl5Q8fZOH9DhDSODiFMvgIbesI7imd5gbaCdpSc", - "VYoZIM8Vmy/QC2GMMoaSMo5QrzIF4l8y7wKlmocVTgeSM7uAnJn//q9r4B3D1hPks46PiNPJRXPRvY2A", - "BAdKc8OubeZMRY4UcEl0xcH438IRi0kxmlHmVjQ/KoohepImP9dQ2x9U5Qt23fnp/LM7foSSYd2+P6T3", - "wP52p9RIolH38iRNltQmeaOZVCOMZHTUwf8Ac6YNKCicMR6aHFoUmHhFBYpTbS4tUfqVk47zZvnVZnPO", - "qUEliXt3OTNLqja4/q1016HUqm/jai+bKkjfld5ZKPhNVZuGFmlD1G71JhAjTXIXGlsok3UqdyizAaOY", - "TT+DvFbMrDb4u62d2G3e62xBSyqeLiC/knWkmIIJjZwRK4yuYGMWwBQ5++Z499EByXGjrsuUaPaLjX+z", - "lQHtwscCNIJAuMydgfE5Ve5va3OBNXOj4Gfrw/LkKNl7lE33n+zku4fZdG9vr9iZZfuPZvn08PETurOb", - "0+lBtlMc7E+L3UcHTw4fT7PH08MCHk33i8Pp7hOYIpfQnDeJ7ngukQnJ0c7+7j46P7zlYC873M0P9rIn", - "+7v7s2JnL3uydzidZQfT6cGT6eNpvkd3Hh3uHOazPVrs7+8e7D3Kdh4f5gf08ZNH08Mn/hYu53NMlJor", - "dg+t8e8zzqLXkeA17xthQp/YwWciU6JliuEJzxnv0N2nT0MN3bCxx9tIQLUeNFnkEY6YxN0ayiJpNosp", - "AtMa+DUa2ecuWUfCBGFzO4k2UtE5DONWPXAYtbgScimsK+eSFs684wFQROzxGgbB6W9E4ge3uAwV/1+l", - "b1a/emx5WBW6f3X53bTjXoX8/gS8G5JHE/bWgrX1XczPQv6xJgBlJ9h8uPDNv9j78G/k73/58NcPf/vw", - "Hx/++ve/fPjPD3/78O/d9sbRo2k/+fe3XOZlkRwl7/2fN+hJF7W4ukR2JEd7iJNRNDeXtC6YDKEfMtda", - "saNkouzOiZ5N3soMAwkQsLO7N7ZHdkP60++/xj8rnRzt7qfJDJNMnRwlO6MdlHFW0jnoS6kur1kBEnlv", - "nyRpImtT1cYVl+CdAeHytmRcWdvgILh0q/oguUsaoDryoRmyauQRH7ktyUAduny8I+dp8otte2ZNdRSZ", - "E2mgddh1V7oVlnaqt7fbSR9U+a5WA1VMNzotuo+I65sIvgm5MQZrI/xIvO5j/VjMjTC8spldpFjYvCO2", - "gCwMJlnUl0pQR11O6HoQzk+9rqfT3QPC5dz7LNu9ZeYL7QsuvtexFtd3wvY+DC8FjDgTvtwvCpbjhcsF", - "xRPzpmy7sPVVzP6Cb7EXj8nLa1BLtA2aVAqumaw1XzlcwqVNphmzqlxGOo3fyjlBoDrtJbwtJUvGOeak", - "odqLQFtS2AuBKs5cjWkY3PdkYdvGbizRdNxxuZSiJl66+fWZEOQKTPzVb8xo1gMOd1MvGYle0UlmLjbS", - "44zNxcuPpURIbi43V7TuHe1OYrYB2wFUt2BtqIGnCyrmMETdaexlayg+KoONhoedw7YCqtgE1T3AcgcE", - "faOrDVXGBcR0Sa9sWqw5QIXBh01T00QvalO4ANqA9qvlbIaWIB4/a5/oniHUDr2lBeCS1ujjB4VDDQp5", - "j+YWTZhbTE6epaSiWi+lKsIrpx1uTIFQE5aqjtqjnbH0sh02qlneGp6FMVVygzAyMZOuyiwMzU1b2G0K", - "wOQcKCpfrbjfqY8mk1kIz5icDOt5P7j+4XOqSlK6FgI5Pj1J0oSzHISGzj1fn357vTc4f7lcjueixmht", - "4vfoybzio73xdAxivDClK7Qxw3vQ+uuSTh062RlPx1NcLSsQtGIY2tlH6BvNwnJmQitmIy0rk1JbUqBk", - "WmKeFK6HWDLjSrpe0v8ki1UgHwi7h1YVZ64yMHmrndVwcnuXVPfr1zcDqtr+lvRhctIVeowerRboSiKl", - "8Kbd6fTeILsFoCXVRNd5DnpWc74ibrrCjkJ4l33NippyN5AxXhtxuRfoXCEpAp99QUKdyKpkXZZUrRpm", - "EkoELG0LDH15I0W+79VpFFm3TTFqtJ0pnVz0jnsRGuluLgREUUkmjMW3Ea1J4x3mEJGvr8E03boHZOaw", - "NRghXbOobQ+uEfBrMIQPWoi2u2ZT+36H9RbStVc15H/bzm316Pf+rcwuWXGzkYTPweQLp6HdBt1P7xOG", - "WPkKkbc87rCBIqUdOt5VXL34fZTOWu0+Oyzm9gWhmZtwsbzbQm7dJlF421ki5IHsndBnk8z+uWnjPRgp", - "1puREbII5BQnAYSIsCJBGgnzeDXTQd81biMQCzPUNWK58MG1nmrtx5qMdDUa9xfTmFjUFE0hba/ToK4x", - "9A9kdf56onzLY7RsOx5R1xN6I74z8jD+J5I6RAjdpn8B+k/qigZdom1k4RP6nFrAuwpyAwUBv6YrQgF8", - "73iWgZ9B6vyDi8gmxxKU2HanXpcozeZiJGezW6IYTIVms6G67g8j0s+PkD6ktia9F0z/dIHGuKXZd1Rd", - "daNoqkkI1u+g9lPKfUM56Dum8d6AhMDgStjJOlh9oYDMpZs4tseP4ywRd3BEPKhS+ys2q3NTj/uUujzM", - "Uv8hlHlrGTyuzQKEcUUrXxpDaQhNnGUzdHTPAqmAFitchee5rmWvXMdahg/F1fhqYNTfd1iW/N6SYSEl", - "uX1P2tLDTbrJmJHNOz5vkfp48XAhyTKMAi9AgRvXXW0gQlwORnmnUBM1XpGizoMasu5FEfJ+37hGh+cW", - "9ux/l9/z9tzzzRFhTM4xNs3tRxOZHfGlORoMDoWL912x3tuStnnQk5WUSIWWK1Al2BdQIy5zyq1po1zf", - "tz27hh42tR6IqvGfkm1wr/kCiprDuRthebi8uvthW4Sx9pO2bkFhk6H6XvqvV/qD6Da/CHOqN2myP927", - "v9JTbyYnAvwpqFDbeAaCOaO5P30S+X7KCSDTREgTPJ3rajlxSomW4bX9CAh6A7kOddvJJUIuHaq7e5/W", - "tQQtogKhlJmhTNiw20KXkqw2bm5+Lu23TEJaO+u07SM19qU7nTbnd6hxlypZmdJewFWk7NTRkMl720fw", - "5ZO4rnT6gdtUUPyBv72Ecv/uooPJJl308RATDsRQw/hob3G+gHDW0prWHKrgUaMqcu77k9Yje6vRFSPH", - "NKsnpn+21Znu+f8obulV2yp2vVKzqlhuyyTdzm6l5FyB1qmf+PWfcCkyo4zXCu70LcGjaBBFrxqG5A6n", - "oxXDiMipibbzS5MwRjZxM8+T9+HBybNbFGZt0HAbpWnPvVVvPpGerCEQ4XxvdqsJLWVtxr9OXcJdVqD9", - "gHm3hdDVm4eV6QYSyl3GZL8e1ePPqWD01H1B2fnCzOVzelVyJlyJIzDB88h1XgxGfeAfWV9FOScLeg3u", - "a1o3BegMj58mzGBmh+0p5803ua1LaTXPacya5p15gCjRXYGxwPTGY6kCGtc8tTZDuI3K9eYOH1JFehdt", - "qyaftJyyJZx15hmEXEESQ0G6lE+DOXYyAMRPjjrUPi/lsMORhAYB7uJuwQ3ffFdSGU2WC5YvPIeoahC7", - "U7SPMUrFa3LOMFnq5Nf9A9uAXQrfp1fX4bNDt1gKv1YbxnkLQkcf7Hne/+i6vJm8t0/YL3DjtAGJozcp", - "hpsmlgqeegFc80hbT5naf7dgg/vSdflRzisd/msNv0D3st7Ab+TWQIFtbm1nWy8eXNs6s9uxakS/jvD5", - "6U53dq+dMZf9GXPXu3IfbA/V5DYj3cji/20xTGMJgLclIfTV7rN5Zmx3sYAZKNIM6jtXbKlhnfrrZHf6", - "+HXSlmLs5KFNVQVfkQxDAlMrTCvst/oterqJddy4hrPs3kX3GO6SXMq1dGdoWYIUQIBre047fRkD00qL", - "JeACaGFbXJ6E/zpy14yeUjF6hniOXtkDkggNO/8yQIyGUrE5E5TbO/H8MTmZ+fFOLrvjoD4uSJHAYUyT", - "iZzXhftXO1pjbSc2U2+tkReU2RUFZLX7DGYL3F56wEbPPWDJvYT2MjdgRtoooHY+sxmMT15ggGfDLLfT", - "qbNYeRWKZMBeJL7QzZaHjOd3p4/jebAVywVt5bIXnI7J99KmjdT/kx2WISiTGTg+e/n2ctcXTM/XSskc", - "tKVIBiim4XTnl99slMgjgkR442acnLJ2hQkF4f9TlU4GgJbe17E2+5EeR9deWvbOpMpZxlck51K7ksE3", - "5+enKKYC7CeljumhWuIN6YwJphege+YICLyjuSGaluADQiPtKDhuKWSNsZrboDd6uG4VA1e2Mf4QO18W", - "wd/Oz7mxxknSaeMM/kmf/tDOYAiNGQ18Nm7NjB1NGVrEFzILXUZb7vi5BsVAp53BtHRtzmfcm4bSkUOP", - "T0/6o3HdJpMsy1r4mXy0tOugd4731ZqIC3b0Oz49Se1FVnJaHnqErF3Bv9/KrEklded8z6+bi5v/CQAA", - "//+lKaE0fE4AAA==", + "H4sIAAAAAAAC/+Q8224cN5a/QtQskATbN11s2XpajR0nMpJYiOTJArEhsapOddNikRWSpXbHEDAfsX+y", + "O8A+7DztD3j+aHFIVhWriy21Esnj2fGD0epiHR6e+439IclkWUkBwujk8EOiswWU1H480prNBeRnVF/i", + "3znoTLHKMCmSw95TwjShxOAnqgkz+LeCDNgV5CRdEbMA8pNUl6AmySiplKxAGQZ2l0yWJRW5/cwMlPbD", + "vygoksPkD9MOuanHbPrMvZBcjxKzqiA5TKhSdIV/v5Mpvu2/1kYxMfffn1eKScXMKljAhIE5qGaF+zby", + "uqBl/MHNMLWhpr71OEi/U7cST0T15WZE6prl+KCQqqQmOXRfjNYXXo8SBb/UTEGeHP7cLELi+LO0uAVH", + "WKNSQJIQq1HHr7ftvjJ9B5lBBI+uKOM05fBSpqdgDKIzkJxTJuYciHbPiSwIJS9lShCajgjIQrLMfezD", + "+WkBgszZFYgR4axkxsrZFeUsx/9r0MRI/E4D8UAm5JXgK1JrxJEsmVkQRzS7Oe7diuCA+OvClkNBa26G", + "eJ0tgPiHDg+iF3IpPDKk1qDIEnHPwYAqmbD7L5huSDJx4AOY8S3ab6ZGSm5Y5TdiotsI5VEVNAMLFHJm", + "8OgOose/oFzDaEhcswCFSFPO5ZLgq+uIEloYXLMA8k6mZEE1SQEE0XVaMmMgn5CfZM1zwsqKr0gOHNxr", + "nBN4z7QDSPWlJoVUDvQ7mY4IFTkaEFlWjOMaZiZvRCfoqZQcqLAnuqJ8SJ+TlVlIQeB9pUBrJi3xUyC4", + "uqYGcqSRVLk7YMMHsCfps67Fq+XNaCgal7Aa4nCcgzCsYKA8kFbkR6SstUF8asF+qZ0geqa984oQ3QcV", + "g6p5RBeOxIrAe6MooWpel2hhGnlLq9UEX9STU1nCidOt1ZdfkQzZUGvIcWWmgBpwR/X6twpw6FS8syx3", + "ECFWlpAzaoCviAIERag9ag4FEwxfGKEhsNvjliNLE1kbjxFVhmU1p6rlwwZ50HXamM+brG7EUJ36N1tV", + "vzOEM//6FdNsXcmMqm8iECpuX7W8PLw+dgYSidWolSJfcnYJhJI/chAoxDTPx1J8NSGnYBDchWXIhTMz", + "zh9T4WyBoLzdwyyowa1rnosvrEC2lgpEbg2IjhN6zcWgAvhFW7qF045Pa96hTsf4xImDU4iG5+RZrRQI", + "w1dEoh2nDVyrYYEl1xNy8e3R6bdfPz9/cfzd1+cnR2ffXrgoJWcKMiPVilTULMi/kos3yfQP9t+b5ILQ", + "qkKS5u7YIOoSz1cwDue4PhklOVPNR/u196gLqheQn3cr30YUeJPQDA28p0Bw+sBqOPdFNTl+3uizPTYK", + "jReJCflBEgEabZ02qs5MrUCTL6370iOSswy3ooqB/opQBUTXVSWVWT+6R36Ekc3eLh6aS2qSkZWFWw8Z", + "P13j7bs9XZTINPmeCjoH5VwAM1b1aYkGOhIacJoCv1vI5om5fbgZC2kG0cCaOniRcOgFe96mG0itiHH/", + "jmnTCIOV7s10G9KoCeN+24nPehZxw3G7LWIHbOL1wbH8A6IAvbR1WZRoFxz6KNNaoveQ1QZuyyM2B+mt", + "AAWPG/TijAteiZ3oa6WkQmDrmUwOvei80ZhhalCC1nQew3cNIQuzWx/D5gWnJYhM/gmU9sHilpS56t64", + "GYtmoderGBYvXepFOX9VJIc/3yxhp018iG9djwaEtLFITGLwgY3mWAna0LJCe9SQO6cGxvgkFjqxCLjX", + "r4+fN27mpc2Obkmsts3p0FS0KV1d5fd8mjXuWEwbmnX7tci+vX7rGPQ9GJpTQy2j8tyGXZSf9Gg/OPFa", + "nKlSZhRVK1J6YN7t6gn5XiqruBWH96HPyahAr1VKjP+txapRy8kFnaST7IIIaRwdmjD5EmzoCe8pwvIC", + "bQXtMDmtFDNAXig2X6AXwhhlAiVlHLFepQrEv6XeBUo1b1Y4HUhO7QJyav73f66AB4atJ8ingY+I08lF", + "c9F3WwFpHCjNDLuymTMVGVLAJdEVB+M/C0csJsW4oMytaD9UFEP0ZJT8UkNtP1CVLdhV8NH5Zwd+jJJh", + "3b4H0vvCfnZQaiTRONw8GSVLapO8cSHVGCMZHXXwP8KcaQMKcmeMhyaH5jkmXlGB4lSbc0uUfuUkcN4s", + "u9xszjk1qCRx7y4Ls6Rqg+vfSnfdkTr1bV3teVsF6bvSWwsFv6tq09Ji1BI1rN40xBglmQuNLZbJOpUD", + "ymw4Ucymn0JWK2ZWG/zd1k7sJu91uqAlFc8WkF3KOlJMwYRGFsQKoyvYmAUwRU6/Pdp99Jhk+KKuyxHR", + "7Fcb/6YrA9qFjzloRIFwmTkD43OqzO/W5QJr5saJPnoxG8kfJl2aOplLJOGCJofJ3qN0tv90J9s9SGd7", + "e3v5TpHuPyqy2cGTp3RnN6Ozx+lO/nh/lu8+evz04MksfTI7yOHRbD8/mO0+hRkCYr9Ccrizv7tv3aDb", + "jcv5HNOdYKvHe+nBbvZ4L326v7tf5Dt76dO9g1mRPp7NHj+dPZlle3Tn0cHOQVbs0Xx/f/fx3qN058lB", + "9pg+efpodvC022r3wLqDPiv9gbcMEB3PXjAOpxVkPzGzOEHEbwsT1w3KuhA0ACNCUEHGCua5aPNhBNbw", + "071OtJGKzmEYGloSDmD2BaiJAxBu1PFb6q0DQZT7khcJ99bIgNh4eLfToiXu9rFVn5bD6Kry4NZyDNlR", + "t6HDur7carNc3mvla3iMzjPe00G2s+cBGG/T1/kRRkmD1YEjr8WlkEthQywuae7cLkod5FE/6YD96Pay", + "NbkfnS/+zWbOmrWevG20XA9koj57c/S7zVCfX7qSQkO8Au+4VShZEkpU8BrxMdcoZKWTMyL7NgvUFagJ", + "eWFB2XocVUCsoGHy65fhd/A+43UOudsQYSiP3aeUgS6MavXhYcQi3KhVtwdwXUHv7TdKTZjMRktdXfTQ", + "dUaQuU3mvsa/MkjTHi7x8Q/2Pv4H+dufP/7l418//tfHv/ztzx//++NfP/5n2Bg8fDTrl838LudZmSeH", + "yQf/5zXGoItaXJ47Fu7hmYyimTmndc5kkzQhIX2cM1X2zakupu9kikwCATu7exMLMkyGT374Bv+sdHKI", + "IlgoWiLTk53xDoonK+kc9LlU51csB4nO136TjBJZm6o2riwL7w0IV/FIJpW13g6Dc7eqj5LbpEUqkBDN", + "kFVjf/CxeyUZSGbIx1uqBW1mvm23ue0rIHMireeAXbcVKpqlQd/jZi/v0xHfD26xiulG0Ny+Q0bc5r5t", + "sorZS5cbRzJdnyXHvDDi8NrWRCJWvH1GbOtFGJKuCPVFRtRRV01x3TtnwN/Us9nuY8LlXLtQ1M49MPOF", + "9qVK3yVcy4iDhLePwysBY86Eb5SJHMNcIMsFRYhZ2/BY2M4EE/PWp9iNJ+TVFagl2gZNKgVXTNaar9xZ", + "mk3bGk0ssuVyHgsH5wSRChqzuBs6NM5JCm2fBJG2pLAbAlWcuersMC3uycK2IxGxEo3jjqtCKGriRc/f", + "XkOATIGJP/qdtYD1yNPt1Evjo1sEZYC3G+lxyubi1V0p0ZQFzjfXgu/92EFJY8NpB1jdcGpDDTxbUDGH", + "4dGdxp53huJOtZ9onhAA2wqpfBNW94DLLRj0ja42VBmXstAlvbQFJc0BKgw+bIFnlOhFbXKX4hjQfrUs", + "CrQEEdvqlMWWiE4Ra3e8pUXgnNaxNPO1BoW8R3OLJswtJsfPR6SiWi+lyptHTjvcgA+hplmqArVHO2Pp", + "ZXvTVLOsMzwLY6rkGnFkopCuPyMMzUzXEmlbJ+QMKCpfrbh/Ux9Op0UTnjE5HVbCf3Sd9xdUlaR0zTdy", + "dHKcjBLOMvA5g9/nm5PvrvYG8JfL5WQuaozWpv4dPZ1XfLw3mU1ATBamdCVqZngPW79dEnRwkp3JbDLD", + "1bICQSuGoZ39amSzcsuZKa2YjbSsTEqXh6JkWmIe5677XjLjmiFe0v8o81VDPhD2HVpV3Fdjpu+0sxpO", + "bm+Nv3udn+sBVW1nWPowOQmFHqNHqwUu57Fn2J3N7g2zGxBaUk10nWWgi5rzFXFzSXaIyLvsK5bXlLtR", + "psnacNi9YOdKsBH87APSVFitStZlSdWqZSahRMDSNo/Rl7dS5DvGQYvVum2KUaPt6erkbQ/cy2YExU1U", + "gcgryYSx521Fa9p6hzlE5OsbMG2f+wGZOWyqR0jXLuoa62sE/AYM4YPmu+1L25S+P5twA+m6rVryv+sm", + "Hnv0+/BOpucsv95IwhdgsoXT0LC1/fOHhOGp/GiKtzwO2ECRRgEdb2tLvP37KJ212n122JPbB4SmbjbM", + "8m4LuXUvidzbzhIxb8gehD6bZPZPbQP8wUix3saPkEUgp3hb9okIKxKklTB/rnau7vvWbTTEwgx1jVgu", + "fHBN21r7gUAjXQXS/cU0JhY1RVNIu+18Kaslq/PXU+WbheNl1yuMup6mq+h7ig/jfyKpQ4TQXfrXYP9J", + "XdGgv7qNLHxCn1MLeF9BZiAn4NeEItSg7x3PsuFnI3X+i7eRl1Tbg+je1OsSpdlcjGVR3BDFYCpUFEN1", + "3R9GpJ8fIX1IbU16L5j++S0a445m31N1GUbRVJMmWL+F2s8o96MYjb5jGu8NSBMYXAo7kwqrLxSQuXSz", + "+hb8JM4ScQtHxIMqtd9iszq39bhPqcvDLPUfQpm3lsGj2ixAGFe08qUxlIamd7hsx/XuWSAV0HyFqxCe", + "6/f3ynWsY/hQXI2vBkb9fcCy5O8tGRZTktnnpCs9XI82GTOy+Y3PW6TuLh4uJFk2Q/QLUOAG3VcbiBCX", + "g3EWFGqixitS1HlQQxZuFCHvD61rdOfcwp79//J73p57vjkiTMgZxqaZvW6U2uF4mqHB4JC7eN8V670t", + "6ZoHPVkZEanQcjVUaewLqDGXGeXWtFGu79ueXUHvNLUeiKrxlzA3uNdsAXnN4cwNfz1cXh1eCY0w1l4G", + "DQsKmwzVD9Lf++pf4bD5RTPhfT1K9md791d66k2zRZA/AdXUNp6DYM5o7s+eRm4eOgFkmghpGk/nulpO", + "nEZEy+axvT4HvVF2d3TbySVCLt1Rd/c+rWtptIgKxFKmhjJhw26L3YiktXE3TubS3gIU0tpZp2131NhX", + "Djpt4QfUuE2VrExpL+AqUnYKNGT6wfYRfPkkritBP3CbCooH+PtLKPfvLoKTbNJFHw8x4VBsahh39hZn", + "C2hgLa1pzaBqPGpURc58f9J6ZG81QjFyTLN6Yvqwrc6E8P9R3NLrrlXseqVmVbHMlknCzm6l5FyB1iM/", + "K+8vPypSUMZrBbf6lsajaBB5rxqG5G6goxXDiMipibZjL9NmkG/qbgtMPzRfHD+/QWHWRnS3UZoO7o16", + "84n0ZO0AEc73xu/a0FLWZvLb1KXZywq0v5oRthBCvXlYmW4xodxlTPbetfYuZ//hETizcfkS/3PktT5W", + "zCfktQZyoXu8CWfvLpARBZvXCogl5YJlCyIF6MnnVO165i5OBxdLXTKqVyVn4tIP6jkJ8hRwbSODIWtL", + "FHS0lHOyoFfgLtG7qTdnNf0YYAqFvWNDOW+v4nf+sDMbjqhrZuPUI0SJDqXdItObiqcKaNxshDOO29qL", + "kKXJQ+p3bM52W1X/pCWhG8ZMY/jWqecXMgkpDnlv2HTUuBYnEkD8XKY74uelK3aMmdBGnkMaWHSbX36o", + "pDLaa7zjFFXtwW6V9COMuHGbjDNM/IJaQR9gl3z4qVzXw3BYdPbGXY82jPMOhUA9LDzvS3VdXk8/2G/Y", + "r3DtlAOJozfpiZs1lwqeeUFc865b31qwv16ywRXruryTIx4Nf7PlV1i/GtDOn0d2bSiwza7dRYnf22pc", + "w9kPXN/5usj93isYzOhEKjr9Wsznp7Ph/GNHz+gou/u5iKF63uQrWh345xb/USyJ8jasSR/8JQFmbIc2", + "hwIUaefjXURgqWFjizfJ7uzJm6QrZ9npTZvuC74iKUYmplaYmtlfCumOp9t40Y28tBcSBgx3hQLKtXQw", + "tCxBCiDAtYXTTbDG0LTSYgm4AJrbNqEn4b+P3TbjZ1SMn+M5x68tgCRCw+B3SWI0lIrNmaDc7onwJ+S4", + "8COyXIYjte3FDWbaUVcm/MULFjoJO/U68l4CeUGZXZFDWrvre1uc7ZVHbPzCI5bcS3okMwNmrI0Camdc", + "28sFyUuMM22059506ixWXoWi4bt99oVuX3nInGh39mTzXRv7A1ONXPZi5An5QdrUm/ofDLIMQZlMwfHZ", + "y7eXu75ger5WSmagLUVSQDFtoLt44GKjRB4SJMKFmxNzyhoKEwrCpjLJjenZzkGUFsrXYDCTKqnJFiQF", + "swTobRrMJjUDS35qwxHAXiKTamBK26zDC8Tn44isg/AlxM3upycIaw+tVBRSZSzlK5JxqV215tuzsxOU", + "bgH2HryTlaZQ5e1vwQTTC9A9KwYE3tPMEE1L8PGrkXYKH1/JZY2hpXtBb3SMYQEJV3apyfB0viKFn517", + "dBOl0yTooA1+h6w/LzWY/2NGAy8mnXWyU0FDQ/pSpk2D11aafqlBMdCjYCZwtDZiNekNoukI0KOT4/5U", + "Ytjfk2VZC38dAg30OuoBeF8oi3huR7+jk+OR3chKTsdDfyBrjvDvdzJtE2EdwPf8un57/X8BAAD//wca", + "4NAxUwAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/pkg/api/openapi_types.gen.go b/pkg/api/openapi_types.gen.go index 4e356b63..f00ead26 100644 --- a/pkg/api/openapi_types.gen.go +++ b/pkg/api/openapi_types.gen.go @@ -66,13 +66,13 @@ const ( JobStatusWaitingForFiles JobStatus = "waiting-for-files" ) -// Defines values for ShamanFileStatusStatus. +// Defines values for ShamanFileStatus. const ( - ShamanFileStatusStatusStored ShamanFileStatusStatus = "stored" + ShamanFileStatusStored ShamanFileStatus = "stored" - ShamanFileStatusStatusUnknown ShamanFileStatusStatus = "unknown" + ShamanFileStatusUnknown ShamanFileStatus = "unknown" - ShamanFileStatusStatusUploading ShamanFileStatusStatus = "uploading" + ShamanFileStatusUploading ShamanFileStatus = "uploading" ) // Defines values for TaskStatus. @@ -244,35 +244,46 @@ type SecurityError struct { // Set of files with their SHA256 checksum, size in bytes, and desired location in the checkout directory. type ShamanCheckout struct { - Req []struct { - // SHA256 checksum of the file - C string `json:"c"` - - // File checkout path - P string `json:"p"` - - // File size in bytes - S int `json:"s"` - } `json:"req"` + Files []ShamanFileSpecWithPath `json:"files"` } -// Status of a file in the Shaman storage. -type ShamanFileStatus struct { - Status ShamanFileStatusStatus `json:"status"` +// Specification of a file in the Shaman storage. +type ShamanFileSpec struct { + // SHA256 checksum of the file + Sha string `json:"sha"` + + // File size in bytes + Size int `json:"size"` } -// ShamanFileStatusStatus defines model for ShamanFileStatus.Status. -type ShamanFileStatusStatus string +// ShamanFileSpecWithPath defines model for ShamanFileSpecWithPath. +type ShamanFileSpecWithPath struct { + // Embedded struct due to allOf(#/components/schemas/ShamanFileSpec) + ShamanFileSpec `yaml:",inline"` + // Embedded fields due to inline allOf schema + // Location of the file in the checkout + Path string `json:"path"` +} + +// ShamanFileSpecWithStatus defines model for ShamanFileSpecWithStatus. +type ShamanFileSpecWithStatus struct { + // Embedded struct due to allOf(#/components/schemas/ShamanFileSpec) + ShamanFileSpec `yaml:",inline"` + // Embedded fields due to inline allOf schema + Status ShamanFileStatus `json:"status"` +} + +// ShamanFileStatus defines model for ShamanFileStatus. +type ShamanFileStatus string // Set of files with their SHA256 checksum and size in bytes. -type ShamanRequirements struct { - Req []struct { - // SHA256 checksum of the file - C string `json:"c"` +type ShamanRequirementsRequest struct { + Files []ShamanFileSpec `json:"files"` +} - // File size in bytes - S int `json:"s"` - } `json:"req"` +// The files from a requirements request, with their status on the Shaman server. Files that are known to Shaman are excluded from the response. +type ShamanRequirementsResponse struct { + Files []ShamanFileSpecWithStatus `json:"files"` } // Job definition submitted to Flamenco. @@ -345,7 +356,7 @@ type TaskUpdateJSONBody TaskUpdate type ShamanCheckoutJSONBody ShamanCheckout // ShamanCheckoutRequirementsJSONBody defines parameters for ShamanCheckoutRequirements. -type ShamanCheckoutRequirementsJSONBody ShamanRequirements +type ShamanCheckoutRequirementsJSONBody ShamanRequirementsRequest // ShamanFileStoreParams defines parameters for ShamanFileStore. type ShamanFileStoreParams struct { diff --git a/pkg/shaman/checkout/checkout.go b/pkg/shaman/checkout/checkout.go new file mode 100644 index 00000000..bf861803 --- /dev/null +++ b/pkg/shaman/checkout/checkout.go @@ -0,0 +1,52 @@ +package checkout + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "context" + "errors" + "fmt" + + "git.blender.org/flamenco/pkg/api" + "git.blender.org/flamenco/pkg/shaman/filestore" + "github.com/rs/zerolog" +) + +var ( + ErrMissingFiles = errors.New("unknown files requested in checkout") +) + +func (m *Manager) Checkout(ctx context.Context, checkoutID string, checkout api.ShamanCheckout) error { + logger := *zerolog.Ctx(ctx) + logger.Debug().Msg("shaman: user requested checkout creation") + + // Actually create the checkout. + resolvedCheckoutInfo, err := m.PrepareCheckout(checkoutID) + if err != nil { + return err + } + + // The checkout directory was created, so if anything fails now, it should be erased. + var checkoutOK bool + defer func() { + if !checkoutOK { + m.EraseCheckout(checkoutID) + } + }() + + for _, fileSpec := range checkout.Files { + blobPath, status := m.fileStore.ResolveFile(fileSpec.Sha, int64(fileSpec.Size), filestore.ResolveStoredOnly) + if status != filestore.StatusStored { + // Caller should upload this file before we can create the checkout. + return ErrMissingFiles + } + + if err := m.SymlinkToCheckout(blobPath, resolvedCheckoutInfo.absolutePath, fileSpec.Path); err != nil { + return fmt.Errorf("symlinking %q to checkout: %w", fileSpec.Path, err) + } + } + + checkoutOK = true // Prevent the checkout directory from being erased again. + logger.Info().Msg("shaman: checkout created") + return nil +} diff --git a/pkg/shaman/checkout/definition.go b/pkg/shaman/checkout/definition.go deleted file mode 100644 index 335b1f74..00000000 --- a/pkg/shaman/checkout/definition.go +++ /dev/null @@ -1,168 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package checkout - -import ( - "bufio" - "context" - "fmt" - "io" - "path" - "regexp" - "strconv" - "strings" - - "github.com/sirupsen/logrus" -) - -/* Checkout Definition files contain a line for each to-be-checked-out file. - * Each line consists of three fields: checksum, file size, path in the checkout. - */ - -// FileInvalidError is returned when there is an invalid line in a checkout definition file. -type FileInvalidError struct { - lineNumber int // base-1 line number that's bad - innerErr error - reason string -} - -func (cfie FileInvalidError) Error() string { - return fmt.Sprintf("invalid line %d: %s", cfie.lineNumber, cfie.reason) -} - -// DefinitionLine is a single line in a checkout definition file. -type DefinitionLine struct { - Checksum string - FileSize int64 - FilePath string -} - -// DefinitionReader reads and parses a checkout definition -type DefinitionReader struct { - ctx context.Context - channel chan *DefinitionLine - reader *bufio.Reader - - Err error - LineNumber int -} - -var ( - // This is a wider range than used in SHA256 sums, but there is no harm in accepting a few more ASCII letters. - validChecksumRegexp = regexp.MustCompile("^[a-zA-Z0-9]{16,}$") -) - -// NewDefinitionReader creates a new DefinitionReader for the given reader. -func NewDefinitionReader(ctx context.Context, reader io.Reader) *DefinitionReader { - return &DefinitionReader{ - ctx: ctx, - channel: make(chan *DefinitionLine), - reader: bufio.NewReader(reader), - } -} - -// Read spins up a new goroutine for parsing the checkout definition. -// The returned channel will receive definition lines. -func (fr *DefinitionReader) Read() <-chan *DefinitionLine { - go func() { - defer close(fr.channel) - defer logrus.Debug("done reading request") - - for { - line, err := fr.reader.ReadString('\n') - if err != nil && err != io.EOF { - fr.Err = FileInvalidError{ - lineNumber: fr.LineNumber, - innerErr: err, - reason: fmt.Sprintf("I/O error: %v", err), - } - return - } - if err == io.EOF && line == "" { - return - } - - if contextError := fr.ctx.Err(); contextError != nil { - fr.Err = fr.ctx.Err() - return - } - - fr.LineNumber++ - logrus.WithFields(logrus.Fields{ - "line": line, - "number": fr.LineNumber, - }).Debug("read line") - - line = strings.TrimSpace(line) - if line == "" { - continue - } - - definitionLine, err := fr.parseLine(line) - if err != nil { - fr.Err = err - return - } - - fr.channel <- definitionLine - } - }() - - return fr.channel -} - -func (fr *DefinitionReader) parseLine(line string) (*DefinitionLine, error) { - - parts := strings.SplitN(strings.TrimSpace(line), " ", 3) - if len(parts) != 3 { - return nil, FileInvalidError{ - lineNumber: fr.LineNumber, - reason: fmt.Sprintf("line should consist of three space-separated parts, not %d: %v", - len(parts), line), - } - } - - checksum := parts[0] - if !validChecksumRegexp.MatchString(checksum) { - return nil, FileInvalidError{fr.LineNumber, nil, "invalid checksum"} - } - - fileSize, err := strconv.ParseInt(parts[1], 10, 64) - if err != nil { - return nil, FileInvalidError{fr.LineNumber, err, "invalid file size"} - } - - filePath := strings.TrimSpace(parts[2]) - if path.IsAbs(filePath) { - return nil, FileInvalidError{fr.LineNumber, err, "no absolute paths allowed"} - } - if filePath != path.Clean(filePath) || strings.Contains(filePath, "..") { - return nil, FileInvalidError{fr.LineNumber, err, "paths must be clean and not have any .. in them."} - } - - return &DefinitionLine{ - Checksum: parts[0], - FileSize: fileSize, - FilePath: filePath, - }, nil -} diff --git a/pkg/shaman/checkout/definition_test.go b/pkg/shaman/checkout/definition_test.go deleted file mode 100644 index 58a2039a..00000000 --- a/pkg/shaman/checkout/definition_test.go +++ /dev/null @@ -1,86 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package checkout - -import ( - "bytes" - "context" - "os" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestDefinitionReader(t *testing.T) { - file, err := os.Open("definition_test_example.txt") - if err != nil { - panic(err) - } - defer file.Close() - - ctx, cancelFunc := context.WithCancel(context.Background()) - reader := NewDefinitionReader(ctx, file) - readChan := reader.Read() - - line := <-readChan - assert.Equal(t, "35b0491c27b0333d1fb45fc0789a12ca06b1d640d2569780b807de504d7029e0", line.Checksum) - assert.Equal(t, int64(1424), line.FileSize) - assert.Equal(t, "definition.go", line.FilePath) - - line = <-readChan - assert.Equal(t, "63b72c63b9424fd13b9370fb60069080c3a15717cf3ad442635b187c6a895079", line.Checksum) - assert.Equal(t, int64(127), line.FileSize) - assert.Equal(t, "logging.go", line.FilePath) - assert.Nil(t, reader.Err) - - // Cancelling is only found out after the next read. - cancelFunc() - line = <-readChan - assert.Nil(t, line) - assert.Equal(t, context.Canceled, reader.Err) - assert.Equal(t, 2, reader.LineNumber) -} - -func TestDefinitionReaderBadRequests(t *testing.T) { - ctx := context.Background() - - testRejects := func(checksum, path string) { - buffer := bytes.NewReader([]byte(checksum + " 30 " + path)) - reader := NewDefinitionReader(ctx, buffer) - readChan := reader.Read() - - var line *DefinitionLine - line = <-readChan - assert.Nil(t, line) - assert.NotNil(t, reader.Err) - assert.Equal(t, 1, reader.LineNumber) - } - - testRejects("35b0491c27b0333d1fb45fc0789a12c", "/etc/passwd") // absolute - testRejects("35b0491c27b0333d1fb45fc0789a12c", "../../../../../../etc/passwd") // ../ in there that path.Clean() will keep - testRejects("35b0491c27b0333d1fb45fc0789a12c", "some/path/../etc/passwd") // ../ in there that path.Clean() will remove - - testRejects("35b", "some/path") // checksum way too short - testRejects("35b0491c.7b0333d1fb45fc0789a12c", "some/path") // checksum invalid - testRejects("35b0491c/7b0333d1fb45fc0789a12c", "some/path") // checksum invalid -} diff --git a/pkg/shaman/checkout/definition_test_example.txt b/pkg/shaman/checkout/definition_test_example.txt deleted file mode 100644 index 9d781163..00000000 --- a/pkg/shaman/checkout/definition_test_example.txt +++ /dev/null @@ -1,5 +0,0 @@ -35b0491c27b0333d1fb45fc0789a12ca06b1d640d2569780b807de504d7029e0 1424 definition.go -63b72c63b9424fd13b9370fb60069080c3a15717cf3ad442635b187c6a895079 127 logging.go -9f1470441beb98dbb66e3339e7da697d9c2312999a6a5610c461cbf55040e210 795 manager.go -59c6bd72af62aa860343adcafd46e3998934a9db2997ce08514b4361f099fa58 1134 routes.go -59c6bd72af62aa860343adcafd46e3998934a9db2997ce08514b4361f099fa58 1134 another-routes.go diff --git a/pkg/shaman/checkout/logging.go b/pkg/shaman/checkout/logging.go deleted file mode 100644 index 84a83858..00000000 --- a/pkg/shaman/checkout/logging.go +++ /dev/null @@ -1,29 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package checkout - -import ( - "github.com/sirupsen/logrus" -) - -var packageLogger = logrus.WithField("package", "shaman/checkout") diff --git a/pkg/shaman/checkout/manager.go b/pkg/shaman/checkout/manager.go index 43f7face..5a4f58d1 100644 --- a/pkg/shaman/checkout/manager.go +++ b/pkg/shaman/checkout/manager.go @@ -30,7 +30,7 @@ import ( "sync" "time" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog/log" "git.blender.org/flamenco/pkg/shaman/config" "git.blender.org/flamenco/pkg/shaman/filestore" @@ -62,12 +62,12 @@ var ( // NewManager creates and returns a new Checkout Manager. func NewManager(conf config.Config, fileStore filestore.Storage) *Manager { - logger := packageLogger.WithField("checkoutDir", conf.CheckoutPath) - logger.Info("opening checkout directory") + logger := log.With().Str("checkoutDir", conf.CheckoutPath).Logger() + logger.Info().Msg("opening checkout directory") err := os.MkdirAll(conf.CheckoutPath, 0777) if err != nil { - logger.WithError(err).Fatal("unable to create checkout directory") + logger.Error().Err(err).Msg("unable to create checkout directory") } return &Manager{conf.CheckoutPath, fileStore, sync.WaitGroup{}} @@ -75,7 +75,7 @@ func NewManager(conf config.Config, fileStore filestore.Storage) *Manager { // Close waits for still-running touch() calls to finish, then returns. func (m *Manager) Close() { - packageLogger.Info("shutting down Checkout manager") + log.Info().Msg("shutting down Checkout manager") m.wg.Wait() } @@ -106,31 +106,31 @@ func (m *Manager) PrepareCheckout(checkoutID string) (ResolvedCheckoutInfo, erro return ResolvedCheckoutInfo{}, err } - logger := logrus.WithFields(logrus.Fields{ - "checkoutPath": checkoutPaths.absolutePath, - "checkoutID": checkoutID, - }) + logger := log.With(). + Str("checkoutPath", checkoutPaths.absolutePath). + Str("checkoutID", checkoutID). + Logger() if stat, err := os.Stat(checkoutPaths.absolutePath); !os.IsNotExist(err) { if err == nil { if stat.IsDir() { - logger.Debug("checkout path exists") + logger.Debug().Msg("checkout path exists") } else { - logger.Error("checkout path exists but is not a directory") + logger.Error().Msg("checkout path exists but is not a directory") } // No error stat'ing this path, indicating it's an existing checkout. return ResolvedCheckoutInfo{}, ErrCheckoutAlreadyExists } // If it's any other error, it's really a problem on our side. - logger.WithError(err).Error("unable to stat checkout directory") + logger.Error().Err(err).Msg("unable to stat checkout directory") return ResolvedCheckoutInfo{}, err } if err := os.MkdirAll(checkoutPaths.absolutePath, 0777); err != nil { - logger.WithError(err).Fatal("unable to create checkout directory") + logger.Error().Err(err).Msg("unable to create checkout directory") } - logger.WithField("relPath", checkoutPaths.RelativePath).Info("created checkout directory") + logger.Info().Str("relPath", checkoutPaths.RelativePath).Msg("created checkout directory") return checkoutPaths, nil } @@ -141,19 +141,19 @@ func (m *Manager) EraseCheckout(checkoutID string) error { return err } - logger := logrus.WithFields(logrus.Fields{ - "checkoutPath": checkoutPaths.absolutePath, - "checkoutID": checkoutID, - }) + logger := log.With(). + Str("checkoutPath", checkoutPaths.absolutePath). + Str("checkoutID", checkoutID). + Logger() if err := os.RemoveAll(checkoutPaths.absolutePath); err != nil { - logger.WithError(err).Error("unable to remove checkout directory") + logger.Error().Err(err).Msg("unable to remove checkout directory") return err } // Try to remove the parent path as well, to not keep the dangling two-letter dirs. // Failure is fine, though, because there is no guarantee it's empty anyway. os.Remove(path.Dir(checkoutPaths.absolutePath)) - logger.Info("removed checkout directory") + logger.Info().Msg("removed checkout directory") return nil } @@ -161,18 +161,18 @@ func (m *Manager) EraseCheckout(checkoutID string) error { // It does *not* do any validation of the validity of the paths! func (m *Manager) SymlinkToCheckout(blobPath, checkoutPath, symlinkRelativePath string) error { symlinkPath := path.Join(checkoutPath, symlinkRelativePath) - logger := logrus.WithFields(logrus.Fields{ - "blobPath": blobPath, - "symlinkPath": symlinkPath, - }) + logger := log.With(). + Str("blobPath", blobPath). + Str("symlinkPath", symlinkPath). + Logger() blobPath, err := filepath.Abs(blobPath) if err != nil { - logger.WithError(err).Error("unable to make blobPath absolute") + logger.Error().Err(err).Msg("unable to make blobPath absolute") return err } - logger.Debug("creating symlink") + logger.Debug().Msg("creating symlink") // This is expected to fail sometimes, because we don't create parent directories yet. // We only create those when we get a failure from symlinking. @@ -181,20 +181,20 @@ func (m *Manager) SymlinkToCheckout(blobPath, checkoutPath, symlinkRelativePath return err } if !os.IsNotExist(err) { - logger.WithError(err).Error("unable to create symlink") + logger.Error().Err(err).Msg("unable to create symlink") return err } - logger.Debug("creating parent directory") + logger.Debug().Msg("creating parent directory") dir := path.Dir(symlinkPath) if err := os.MkdirAll(dir, 0777); err != nil { - logger.WithError(err).Error("unable to create parent directory") + logger.Error().Err(err).Msg("unable to create parent directory") return err } if err := os.Symlink(blobPath, symlinkPath); err != nil { - logger.WithError(err).Error("unable to create symlink, after creating parent directory") + logger.Error().Err(err).Msg("unable to create symlink, after creating parent directory") return err } @@ -215,23 +215,17 @@ func touchFile(blobPath string) error { } now := time.Now() - logger := logrus.WithField("file", blobPath) - logger.Debug("touching") - err := touch.Touch(blobPath) - logLevel := logrus.DebugLevel if err != nil { - logger = logger.WithError(err) - logLevel = logrus.WarnLevel + return err } duration := time.Now().Sub(now) - logger = logger.WithField("duration", duration) - if duration < 1*time.Second { - logger.Log(logLevel, "done touching") - } else { - logger.Log(logLevel, "done touching but took a long time") + logger := log.With().Str("file", blobPath).Logger() + if duration > 1*time.Second { + logger.Warn().Str("duration", duration.String()).Msg("done touching but took a long time") } + logger.Debug().Msg("done touching") return err } diff --git a/pkg/shaman/checkout/report_requirements.go b/pkg/shaman/checkout/report_requirements.go new file mode 100644 index 00000000..a6077870 --- /dev/null +++ b/pkg/shaman/checkout/report_requirements.go @@ -0,0 +1,63 @@ +package checkout + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "context" + "fmt" + + "git.blender.org/flamenco/pkg/api" + "git.blender.org/flamenco/pkg/shaman/filestore" + + "github.com/rs/zerolog" +) + +func (m *Manager) ReportRequirements(ctx context.Context, requirements api.ShamanRequirementsRequest) (api.ShamanRequirementsResponse, error) { + logger := zerolog.Ctx(ctx) + logger.Debug().Msg("user requested checkout requirements") + + missing := api.ShamanRequirementsResponse{} + alreadyRequested := map[string]bool{} + for _, fileSpec := range requirements.Files { + fileKey := fmt.Sprintf("%s/%d", fileSpec.Sha, fileSpec.Size) + if alreadyRequested[fileKey] { + // User asked for this (checksum, filesize) tuple already. + continue + } + + path, status := m.fileStore.ResolveFile(fileSpec.Sha, int64(fileSpec.Size), filestore.ResolveEverything) + + var apiStatus api.ShamanFileStatus + switch status { + case filestore.StatusDoesNotExist: + // Caller can upload this file immediately. + apiStatus = api.ShamanFileStatusUnknown + case filestore.StatusUploading: + // Caller should postpone uploading this file until all 'unknown' files have been uploaded. + apiStatus = api.ShamanFileStatusUploading + case filestore.StatusStored: + // We expect this file to be sent soon, though, so we need to + // 'touch' it to make sure it won't be GC'd in the mean time. + go touchFile(path) + + // Only send a response when the caller needs to do something. + continue + default: + logger.Error(). + Str("path", path). + Str("status", status.String()). + Str("checksum", fileSpec.Sha). + Int("filesize", fileSpec.Size). + Msg("invalid status returned by ResolveFile") + continue + } + + alreadyRequested[fileKey] = true + missing.Files = append(missing.Files, api.ShamanFileSpecWithStatus{ + ShamanFileSpec: fileSpec, + Status: apiStatus, + }) + } + + return missing, nil +} diff --git a/pkg/shaman/checkout/routes.go b/pkg/shaman/checkout/routes.go deleted file mode 100644 index c34de038..00000000 --- a/pkg/shaman/checkout/routes.go +++ /dev/null @@ -1,191 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package checkout - -import ( - "fmt" - "net/http" - "strings" - - "git.blender.org/flamenco/pkg/shaman/filestore" - "git.blender.org/flamenco/pkg/shaman/httpserver" - - "git.blender.org/flamenco/pkg/shaman/jwtauth" - "github.com/gorilla/mux" - "github.com/sirupsen/logrus" -) - -// Responses for each line of a checkout definition file. -const ( - responseFileUnkown = "file-unknown" - responseAlreadyUploading = "already-uploading" - responseError = "ERROR" -) - -// AddRoutes adds HTTP routes to the muxer. -func (m *Manager) AddRoutes(router *mux.Router, auther jwtauth.Authenticator) { - router.Handle("/checkout/requirements", auther.WrapFunc(m.reportRequirements)).Methods("POST") - router.Handle("/checkout/create/{checkoutID}", auther.WrapFunc(m.createCheckout)).Methods("POST") -} - -func (m *Manager) reportRequirements(w http.ResponseWriter, r *http.Request) { - logger := packageLogger.WithFields(jwtauth.RequestLogFields(r)) - logger.Debug("user requested checkout requirements") - - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - if r.Header.Get("Content-Type") != "text/plain" { - http.Error(w, "Expecting text/plain content type", http.StatusBadRequest) - return - } - - bodyReader, err := httpserver.DecompressedReader(r) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - defer bodyReader.Close() - - // Unfortunately, Golang doesn't allow us (for good reason) to send a reply while - // still reading the response. See https://github.com/golang/go/issues/4637 - responseLines := []string{} - alreadyRequested := map[string]bool{} - reader := NewDefinitionReader(r.Context(), bodyReader) - for line := range reader.Read() { - fileKey := fmt.Sprintf("%s/%d", line.Checksum, line.FileSize) - if alreadyRequested[fileKey] { - // User asked for this (checksum, filesize) tuple already. - continue - } - - path, status := m.fileStore.ResolveFile(line.Checksum, line.FileSize, filestore.ResolveEverything) - - response := "" - switch status { - case filestore.StatusDoesNotExist: - // Caller can upload this file immediately. - response = responseFileUnkown - case filestore.StatusUploading: - // Caller should postpone uploading this file until all 'does-not-exist' files have been uploaded. - response = responseAlreadyUploading - case filestore.StatusStored: - // We expect this file to be sent soon, though, so we need to - // 'touch' it to make sure it won't be GC'd in the mean time. - go touchFile(path) - - // Only send a response when the caller needs to do something. - continue - default: - logger.WithFields(logrus.Fields{ - "path": path, - "status": status, - "checksum": line.Checksum, - "filesize": line.FileSize, - }).Error("invalid status returned by ResolveFile") - continue - } - - alreadyRequested[fileKey] = true - responseLines = append(responseLines, fmt.Sprintf("%s %s\n", response, line.FilePath)) - } - if reader.Err != nil { - logger.WithError(reader.Err).Warning("error reading checkout definition") - http.Error(w, fmt.Sprintf("%s %v\n", responseError, reader.Err), http.StatusBadRequest) - return - } - - w.WriteHeader(http.StatusOK) - w.Write([]byte(strings.Join(responseLines, ""))) -} - -func (m *Manager) createCheckout(w http.ResponseWriter, r *http.Request) { - checkoutID := mux.Vars(r)["checkoutID"] - - logger := packageLogger.WithFields(jwtauth.RequestLogFields(r)).WithField("checkoutID", checkoutID) - logger.Debug("user requested checkout creation") - - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - if r.Header.Get("Content-Type") != "text/plain" { - http.Error(w, "Expecting text/plain content type", http.StatusBadRequest) - return - } - bodyReader, err := httpserver.DecompressedReader(r) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - defer bodyReader.Close() - - // Actually create the checkout. - resolvedCheckoutInfo, err := m.PrepareCheckout(checkoutID) - if err != nil { - switch err { - case ErrInvalidCheckoutID: - http.Error(w, fmt.Sprintf("invalid checkout ID '%s'", checkoutID), http.StatusBadRequest) - case ErrCheckoutAlreadyExists: - http.Error(w, fmt.Sprintf("checkout '%s' already exists", checkoutID), http.StatusConflict) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // The checkout directory was created, so if anything fails now, it should be erased. - var checkoutOK bool - defer func() { - if !checkoutOK { - m.EraseCheckout(checkoutID) - } - }() - - responseLines := []string{} - reader := NewDefinitionReader(r.Context(), bodyReader) - for line := range reader.Read() { - blobPath, status := m.fileStore.ResolveFile(line.Checksum, line.FileSize, filestore.ResolveStoredOnly) - if status != filestore.StatusStored { - // Caller should upload this file before we can create the checkout. - responseLines = append(responseLines, fmt.Sprintf("%s %s\n", responseFileUnkown, line.FilePath)) - continue - } - - if err := m.SymlinkToCheckout(blobPath, resolvedCheckoutInfo.absolutePath, line.FilePath); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - if reader.Err != nil { - http.Error(w, fmt.Sprintf("ERROR %v\n", reader.Err), http.StatusBadRequest) - return - } - - // If there was any file missing, we should just stop now. - if len(responseLines) > 0 { - http.Error(w, strings.Join(responseLines, ""), http.StatusBadRequest) - return - } - - w.WriteHeader(http.StatusOK) - w.Write([]byte(resolvedCheckoutInfo.RelativePath)) - - checkoutOK = true // Prevent the checkout directory from being erased again. - logger.Info("checkout created") -} diff --git a/pkg/shaman/checkout/routes_test.go b/pkg/shaman/checkout/routes_test.go index 376d2f34..76c827f2 100644 --- a/pkg/shaman/checkout/routes_test.go +++ b/pkg/shaman/checkout/routes_test.go @@ -23,18 +23,10 @@ package checkout import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "path" - "strings" + "context" "testing" - "git.blender.org/flamenco/pkg/shaman/filestore" - "git.blender.org/flamenco/pkg/shaman/httpserver" - "github.com/gorilla/mux" - "github.com/sirupsen/logrus" + "git.blender.org/flamenco/pkg/api" "github.com/stretchr/testify/assert" ) @@ -42,84 +34,72 @@ func TestReportRequirements(t *testing.T) { manager, cleanup := createTestManager() defer cleanup() - defFile, err := ioutil.ReadFile("definition_test_example.txt") - assert.Nil(t, err) - compressedDefFile := httpserver.CompressBuffer(defFile) - - // 5 files, all ending in newline, so defFileLines has trailing "" element. - defFileLines := strings.Split(string(defFile), "\n") - assert.Equal(t, 6, len(defFileLines), defFileLines) - - respRec := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/checkout/requirement", compressedDefFile) - req.Header.Set("Content-Type", "text/plain") - req.Header.Set("Content-Encoding", "gzip") - manager.reportRequirements(respRec, req) - - bodyBytes, err := ioutil.ReadAll(respRec.Body) - assert.Nil(t, err) - body := string(bodyBytes) - - assert.Equal(t, respRec.Code, http.StatusOK, body) - - // We should not be required to upload the same file twice, - // so another-routes.go should not be in the response. - lines := strings.Split(body, "\n") - expectLines := []string{ - "file-unknown definition.go", - "file-unknown logging.go", - "file-unknown manager.go", - "file-unknown routes.go", - "", + required := api.ShamanRequirementsRequest{ + Files: []api.ShamanFileSpec{ + {"63b72c63b9424fd13b9370fb60069080c3a15717cf3ad442635b187c6a895079", 127}, + {"9f1470441beb98dbb66e3339e7da697d9c2312999a6a5610c461cbf55040e210", 795}, + {"59c6bd72af62aa860343adcafd46e3998934a9db2997ce08514b4361f099fa58", 1134}, + {"59c6bd72af62aa860343adcafd46e3998934a9db2997ce08514b4361f099fa58", 1134}, // duplicate of the above + }, } - assert.EqualValues(t, expectLines, lines) + + response, err := manager.ReportRequirements(context.Background(), required) + assert.NoError(t, err) + + // We should not be required to upload the same file twice, so the duplicate + // should not be in the response. + assert.Equal(t, []api.ShamanFileSpecWithStatus{ + {api.ShamanFileSpec{"63b72c63b9424fd13b9370fb60069080c3a15717cf3ad442635b187c6a895079", 127}, api.ShamanFileStatusUnknown}, + {api.ShamanFileSpec{"9f1470441beb98dbb66e3339e7da697d9c2312999a6a5610c461cbf55040e210", 795}, api.ShamanFileStatusUnknown}, + {api.ShamanFileSpec{"59c6bd72af62aa860343adcafd46e3998934a9db2997ce08514b4361f099fa58", 1134}, api.ShamanFileStatusUnknown}, + }, response.Files) } -func TestCreateCheckout(t *testing.T) { - manager, cleanup := createTestManager() - defer cleanup() +// func TestCreateCheckout(t *testing.T) { +// manager, cleanup := createTestManager() +// defer cleanup() - filestore.LinkTestFileStore(manager.fileStore.BasePath()) +// filestore.LinkTestFileStore(manager.fileStore.BasePath()) - defFile, err := ioutil.ReadFile("../_test_file_store/checkout_definition.txt") - assert.Nil(t, err) - compressedDefFile := httpserver.CompressBuffer(defFile) +// defFile, err := ioutil.ReadFile("../_test_file_store/checkout_definition.txt") +// assert.Nil(t, err) +// compressedDefFile := httpserver.CompressBuffer(defFile) - respRec := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/checkout/create/{checkoutID}", compressedDefFile) - req = mux.SetURLVars(req, map[string]string{ - "checkoutID": "jemoeder", - }) - req.Header.Set("Content-Type", "text/plain") - req.Header.Set("Content-Encoding", "gzip") - logrus.SetLevel(logrus.DebugLevel) - manager.createCheckout(respRec, req) +// respRec := httptest.NewRecorder() +// req := httptest.NewRequest("POST", "/checkout/create/{checkoutID}", compressedDefFile) +// req = mux.SetURLVars(req, map[string]string{ +// "checkoutID": "jemoeder", +// }) +// req.Header.Set("Content-Type", "text/plain") +// req.Header.Set("Content-Encoding", "gzip") +// logrus.SetLevel(logrus.DebugLevel) +// manager.createCheckout(respRec, req) - bodyBytes, err := ioutil.ReadAll(respRec.Body) - assert.Nil(t, err) - body := string(bodyBytes) - assert.Equal(t, http.StatusOK, respRec.Code, body) +// bodyBytes, err := ioutil.ReadAll(respRec.Body) +// assert.Nil(t, err) +// body := string(bodyBytes) +// assert.Equal(t, http.StatusOK, respRec.Code, body) - // Check the symlinks of the checkout - coPath := path.Join(manager.checkoutBasePath, "er", "jemoeder") - assert.FileExists(t, path.Join(coPath, "subdir", "replacer.py")) - assert.FileExists(t, path.Join(coPath, "feed.py")) - assert.FileExists(t, path.Join(coPath, "httpstuff.py")) - assert.FileExists(t, path.Join(coPath, "filesystemstuff.py")) +// // Check the symlinks of the checkout +// coPath := path.Join(manager.checkoutBasePath, "er", "jemoeder") +// assert.FileExists(t, path.Join(coPath, "subdir", "replacer.py")) +// assert.FileExists(t, path.Join(coPath, "feed.py")) +// assert.FileExists(t, path.Join(coPath, "httpstuff.py")) +// assert.FileExists(t, path.Join(coPath, "filesystemstuff.py")) - storePath := manager.fileStore.StoragePath() - assertLinksTo(t, path.Join(coPath, "subdir", "replacer.py"), - path.Join(storePath, "59", "0c148428d5c35fab3ebad2f3365bb469ab9c531b60831f3e826c472027a0b9", "3367.blob")) - assertLinksTo(t, path.Join(coPath, "feed.py"), - path.Join(storePath, "80", "b749c27b2fef7255e7e7b3c2029b03b31299c75ff1f1c72732081c70a713a3", "7488.blob")) - assertLinksTo(t, path.Join(coPath, "httpstuff.py"), - path.Join(storePath, "91", "4853599dd2c351ab7b82b219aae6e527e51518a667f0ff32244b0c94c75688", "486.blob")) - assertLinksTo(t, path.Join(coPath, "filesystemstuff.py"), - path.Join(storePath, "d6", "fc7289b5196cc96748ea72f882a22c39b8833b457fe854ef4c03a01f5db0d3", "7217.blob")) -} +// storePath := manager.fileStore.StoragePath() +// assertLinksTo(t, path.Join(coPath, "subdir", "replacer.py"), +// path.Join(storePath, "59", "0c148428d5c35fab3ebad2f3365bb469ab9c531b60831f3e826c472027a0b9", "3367.blob")) +// assertLinksTo(t, path.Join(coPath, "feed.py"), +// path.Join(storePath, "80", "b749c27b2fef7255e7e7b3c2029b03b31299c75ff1f1c72732081c70a713a3", "7488.blob")) +// assertLinksTo(t, path.Join(coPath, "httpstuff.py"), +// path.Join(storePath, "91", "4853599dd2c351ab7b82b219aae6e527e51518a667f0ff32244b0c94c75688", "486.blob")) +// assertLinksTo(t, path.Join(coPath, "filesystemstuff.py"), +// path.Join(storePath, "d6", "fc7289b5196cc96748ea72f882a22c39b8833b457fe854ef4c03a01f5db0d3", "7217.blob")) +// } -func assertLinksTo(t *testing.T, linkPath, expectedTarget string) { - actualTarget, err := os.Readlink(linkPath) - assert.Nil(t, err) - assert.Equal(t, expectedTarget, actualTarget) -} +// func assertLinksTo(t *testing.T, linkPath, expectedTarget string) { +// actualTarget, err := os.Readlink(linkPath) +// assert.Nil(t, err) +// assert.Equal(t, expectedTarget, actualTarget) +// } diff --git a/pkg/shaman/fileserver/receivefile_test.go b/pkg/shaman/fileserver/receivefile_test.go index e61a35fd..37326287 100644 --- a/pkg/shaman/fileserver/receivefile_test.go +++ b/pkg/shaman/fileserver/receivefile_test.go @@ -23,17 +23,16 @@ package fileserver import ( + "bytes" + "context" + "io" "io/ioutil" - "net/http" - "net/http/httptest" - "strconv" "testing" + "git.blender.org/flamenco/pkg/shaman/config" "git.blender.org/flamenco/pkg/shaman/hasher" - "git.blender.org/flamenco/pkg/shaman/httpserver" "git.blender.org/flamenco/pkg/shaman/filestore" - "github.com/gorilla/mux" "github.com/stretchr/testify/assert" ) @@ -46,37 +45,42 @@ func TestStoreFile(t *testing.T) { assert.EqualValues(t, []byte("h\xc3\xa4h\xc3\xa4h\xc3\xa4"), payload) filesize := int64(len(payload)) + correctChecksum := hasher.Checksum(payload) - testWithChecksum := func(checksum string) *httptest.ResponseRecorder { - compressedPayload := httpserver.CompressBuffer(payload) - respRec := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/files/{checksum}/{filesize}", compressedPayload) - req = mux.SetURLVars(req, map[string]string{ - "checksum": checksum, - "filesize": strconv.FormatInt(filesize, 10), - }) - req.Header.Set("Content-Encoding", "gzip") - req.Header.Set("X-Shaman-Original-Filename", "in-memory-file.txt") - server.ServeHTTP(respRec, req) - return respRec + testWithChecksum := func(checksum string, reportSize int64) error { + buffer := io.NopCloser(bytes.NewBuffer(payload)) + return server.ReceiveFile(context.Background(), buffer, checksum, reportSize, false) } - var respRec *httptest.ResponseRecorder + var err error var path string var status filestore.FileStatus // A bad checksum should be rejected. badChecksum := "da-checksum-is-long-enough-like-this" - respRec = testWithChecksum(badChecksum) - assert.Equal(t, http.StatusExpectationFailed, respRec.Code) + err = testWithChecksum(badChecksum, filesize) + assert.ErrorIs(t, err, ErrFileChecksumMismatch{ + DeclaredChecksum: badChecksum, + ActualChecksum: correctChecksum, + }) + path, status = server.fileStore.ResolveFile(badChecksum, filesize, filestore.ResolveEverything) + assert.Equal(t, filestore.StatusDoesNotExist, status) + assert.Equal(t, "", path) + + // A bad file size should be rejected. + err = testWithChecksum(correctChecksum, filesize+1) + assert.ErrorIs(t, err, ErrFileSizeMismatch{ + DeclaredSize: filesize + 1, + ActualSize: filesize, + }) path, status = server.fileStore.ResolveFile(badChecksum, filesize, filestore.ResolveEverything) assert.Equal(t, filestore.StatusDoesNotExist, status) assert.Equal(t, "", path) // The correct checksum should be accepted. - correctChecksum := hasher.Checksum(payload) - respRec = testWithChecksum(correctChecksum) - assert.Equal(t, http.StatusNoContent, respRec.Code) + err = testWithChecksum(correctChecksum, filesize) + assert.NoError(t, err) + path, status = server.fileStore.ResolveFile(correctChecksum, filesize, filestore.ResolveEverything) assert.Equal(t, filestore.StatusStored, status) assert.FileExists(t, path) @@ -85,3 +89,17 @@ func TestStoreFile(t *testing.T) { assert.Nil(t, err) assert.EqualValues(t, payload, savedContent, "The file should be saved uncompressed") } + +func createTestServer() (server *FileServer, cleanup func()) { + config, configCleanup := config.CreateTestConfig() + + store := filestore.New(config) + server = New(store) + server.Go() + + cleanup = func() { + server.Close() + configCleanup() + } + return +} diff --git a/pkg/shaman/fileserver/receivelistener.go b/pkg/shaman/fileserver/receivelistener.go index cd553ab4..e4639043 100644 --- a/pkg/shaman/fileserver/receivelistener.go +++ b/pkg/shaman/fileserver/receivelistener.go @@ -25,6 +25,8 @@ package fileserver import ( "fmt" "time" + + "github.com/rs/zerolog/log" ) // Returns a channel that is open while the given file is being received. @@ -68,10 +70,10 @@ func (fs *FileServer) receiveListenerPeriodicCheck() { numChans := len(fs.receiverChannels) if numChans == 0 { if lastReportedChans != 0 { - packageLogger.Debug("no receive listener channels") + log.Debug().Msg("no receive listener channels") } } else { - packageLogger.WithField("num_receiver_channels", numChans).Debug("receiving files") + log.Debug().Int("num_receiver_channels", numChans).Msg("receiving files") } lastReportedChans = numChans } @@ -79,7 +81,7 @@ func (fs *FileServer) receiveListenerPeriodicCheck() { for { select { case <-fs.ctx.Done(): - packageLogger.Debug("stopping receive listener periodic check") + log.Debug().Msg("stopping receive listener periodic check") return case <-time.After(1 * time.Minute): doCheck() diff --git a/pkg/shaman/fileserver/servefile.go b/pkg/shaman/fileserver/servefile.go deleted file mode 100644 index a2bdaa13..00000000 --- a/pkg/shaman/fileserver/servefile.go +++ /dev/null @@ -1,83 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package fileserver - -import ( - "context" - "fmt" - "io" - "net/http" - "os" - "strconv" - - "github.com/sirupsen/logrus" - - "git.blender.org/flamenco/pkg/shaman/filestore" -) - -// serveFile only serves stored files (not 'uploading' or 'checking') -func (fs *FileServer) serveFile(ctx context.Context, w http.ResponseWriter, checksum string, filesize int64) { - path, status := fs.fileStore.ResolveFile(checksum, filesize, filestore.ResolveStoredOnly) - if status != filestore.StatusStored { - http.Error(w, "File Not Found", http.StatusNotFound) - return - } - - logger := packageLogger.WithField("path", path) - - stat, err := os.Stat(path) - if err != nil { - logger.WithError(err).Error("unable to stat file") - http.Error(w, "File Not Found", http.StatusNotFound) - return - } - if stat.Size() != filesize { - logger.WithFields(logrus.Fields{ - "realSize": stat.Size(), - "expectedSize": filesize, - }).Error("file size in storage is corrupt") - http.Error(w, "File Size Incorrect", http.StatusInternalServerError) - return - } - - infile, err := os.Open(path) - if err != nil { - logger.WithError(err).Error("unable to read file") - http.Error(w, "File Not Found", http.StatusNotFound) - return - } - - filesizeStr := strconv.FormatInt(filesize, 10) - w.Header().Set("Content-Type", "application/binary") - w.Header().Set("Content-Length", filesizeStr) - w.Header().Set("ETag", fmt.Sprintf("'%s-%s'", checksum, filesizeStr)) - w.Header().Set("X-Shaman-Checksum", checksum) - - written, err := io.Copy(w, infile) - if err != nil { - logger.WithError(err).Error("unable to copy file to writer") - // Anything could have been sent by now, so just close the connection. - return - } - logger.WithField("written", written).Debug("file send to writer") -} diff --git a/pkg/shaman/fileserver/servefile_test.go b/pkg/shaman/fileserver/servefile_test.go deleted file mode 100644 index 9a815cca..00000000 --- a/pkg/shaman/fileserver/servefile_test.go +++ /dev/null @@ -1,71 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package fileserver - -import ( - "net/http" - "net/http/httptest" - "strconv" - "testing" - - "git.blender.org/flamenco/pkg/shaman/config" - "git.blender.org/flamenco/pkg/shaman/filestore" - "github.com/gorilla/mux" - "github.com/stretchr/testify/assert" -) - -func createTestServer() (server *FileServer, cleanup func()) { - config, configCleanup := config.CreateTestConfig() - - store := filestore.New(config) - server = New(store) - server.Go() - - cleanup = func() { - server.Close() - configCleanup() - } - return -} - -func TestServeFile(t *testing.T) { - server, cleanup := createTestServer() - defer cleanup() - - payload := []byte("hähähä") - checksum := "da-checksum-is-long-enough-like-this" - filesize := int64(len(payload)) - - server.fileStore.(*filestore.Store).MustStoreFileForTest(checksum, filesize, payload) - - respRec := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/files/{checksum}/{filesize}", nil) - req = mux.SetURLVars(req, map[string]string{ - "checksum": checksum, - "filesize": strconv.FormatInt(filesize, 10), - }) - server.ServeHTTP(respRec, req) - - assert.Equal(t, http.StatusOK, respRec.Code) - assert.EqualValues(t, payload, respRec.Body.Bytes()) -} diff --git a/pkg/shaman/filestore/interface.go b/pkg/shaman/filestore/interface.go index 9643c5d7..ab9e2a6e 100644 --- a/pkg/shaman/filestore/interface.go +++ b/pkg/shaman/filestore/interface.go @@ -24,6 +24,7 @@ package filestore import ( "errors" + "fmt" "os" ) @@ -65,6 +66,19 @@ const ( StatusStored ) +func (fs FileStatus) String() string { + switch fs { + case StatusDoesNotExist: + return "DoesNotExist" + case StatusUploading: + return "Uploading" + case StatusStored: + return "Stored" + default: + return fmt.Sprintf("invalid(%d)", int(fs)) + } +} + // StoredOnly indicates whether to resolve only 'stored' files or also 'uploading' or 'checking'. type StoredOnly bool diff --git a/pkg/shaman/server.go b/pkg/shaman/server.go index 6def6ee9..7612bd0c 100644 --- a/pkg/shaman/server.go +++ b/pkg/shaman/server.go @@ -104,24 +104,24 @@ func (s *Server) Close() { // Checkout creates a directory, and symlinks the required files into it. The // files must all have been uploaded to Shaman before calling this. func (s *Server) Checkout(ctx context.Context, checkoutID string, checkout api.ShamanCheckout) error { - return nil + return s.checkoutMan.Checkout(ctx, checkoutID, checkout) } // Requirements checks a Shaman Requirements file, and returns the subset // containing the unknown files. -func (s *Server) Requirements(ctx context.Context, requirements api.ShamanRequirements) (api.ShamanRequirements, error) { - return requirements, nil +func (s *Server) Requirements(ctx context.Context, requirements api.ShamanRequirementsRequest) (api.ShamanRequirementsResponse, error) { + return s.checkoutMan.ReportRequirements(ctx, requirements) } -var fsStatusToApiStatus = map[filestore.FileStatus]api.ShamanFileStatusStatus{ - filestore.StatusDoesNotExist: api.ShamanFileStatusStatusUnknown, - filestore.StatusUploading: api.ShamanFileStatusStatusUploading, - filestore.StatusStored: api.ShamanFileStatusStatusStored, +var fsStatusToApiStatus = map[filestore.FileStatus]api.ShamanFileStatus{ + filestore.StatusDoesNotExist: api.ShamanFileStatusUnknown, + filestore.StatusUploading: api.ShamanFileStatusUploading, + filestore.StatusStored: api.ShamanFileStatusStored, } // Check the status of a file on the Shaman server. // status (stored, currently being uploaded, unknown). -func (s *Server) FileStoreCheck(ctx context.Context, checksum string, filesize int64) api.ShamanFileStatusStatus { +func (s *Server) FileStoreCheck(ctx context.Context, checksum string, filesize int64) api.ShamanFileStatus { status := s.fileServer.CheckFile(checksum, filesize) apiStatus, ok := fsStatusToApiStatus[status] if !ok { @@ -130,7 +130,7 @@ func (s *Server) FileStoreCheck(ctx context.Context, checksum string, filesize i Int64("filesize", filesize). Int("fileserverStatus", int(status)). Msg("shaman: unknown status on fileserver") - return api.ShamanFileStatusStatusUnknown + return api.ShamanFileStatusUnknown } return apiStatus }