diff --git a/internal/manager/api_impl/api_impl.go b/internal/manager/api_impl/api_impl.go index 4dd226b6..d06b9e65 100644 --- a/internal/manager/api_impl/api_impl.go +++ b/internal/manager/api_impl/api_impl.go @@ -89,9 +89,7 @@ type Shaman interface { Requirements(ctx context.Context, requirements api.ShamanRequirements) (api.ShamanRequirements, error) // Check the status of a file on the Shaman server. - // TODO: instead of an integer, return a constant that indicates the actual - // status (stored, currently being uploaded, unknown). - FileStoreCheck(ctx context.Context, checksum string, filesize int64) (int, error) + FileStoreCheck(ctx context.Context, checksum string, filesize int64) (api.ShamanFileStatusStatus, error) // Store a new file on the Shaman server. Note that the Shaman server can // return early when another client finishes uploading the exact same file, to diff --git a/internal/manager/api_impl/shaman.go b/internal/manager/api_impl/shaman.go index 94898172..00109bce 100644 --- a/internal/manager/api_impl/shaman.go +++ b/internal/manager/api_impl/shaman.go @@ -8,6 +8,7 @@ import ( "github.com/labstack/echo/v4" "git.blender.org/flamenco/pkg/api" + "git.blender.org/flamenco/pkg/shaman/fileserver" ) // Create a directory, and symlink the required files into it. The files must all have been uploaded to Shaman before calling this endpoint. @@ -70,11 +71,11 @@ func (f *Flamenco) ShamanFileStoreCheck(e echo.Context, checksum string, filesiz // TODO: actually switch over the actual statuses, see the TODO in the Shaman interface. switch status { - case 0: // known + case api.ShamanFileStatusStatusStored: return e.String(http.StatusOK, "") - case 1: // being uploaded + case api.ShamanFileStatusStatusUploading: return e.String(420 /* Enhance Your Calm */, "") - case 2: // unknown + case api.ShamanFileStatusStatusUnknown: return e.String(http.StatusNotFound, "") } @@ -108,7 +109,21 @@ func (f *Flamenco) ShamanFileStore(e echo.Context, checksum string, filesize int canDefer, origFilename, ) if err != nil { - logger.Warn().Err(err).Msg("Shaman: checking stored file") + if err == fileserver.ErrFileAlreadyExists { + return e.String(http.StatusAlreadyReported, "") + } + + logger.Warn().Err(err).Msg("shaman: checking stored file") + if sizeErr, ok := err.(fileserver.ErrFileSizeMismatch); ok { + return sendAPIError(e, http.StatusExpectationFailed, + "size mismatch, expected %d bytes, received %d bytes", + sizeErr.DeclaredSize, sizeErr.ActualSize) + } + if checksumErr, ok := err.(fileserver.ErrFileChecksumMismatch); ok { + return sendAPIError(e, http.StatusExpectationFailed, + "checksum mismatch, expected %d bytes, received %d bytes", + checksumErr.DeclaredChecksum, checksumErr.ActualChecksum) + } return sendAPIError(e, http.StatusInternalServerError, "unexpected error: %v", err) } diff --git a/pkg/api/flamenco-manager.yaml b/pkg/api/flamenco-manager.yaml index ff4c4dd1..31698ca3 100644 --- a/pkg/api/flamenco-manager.yaml +++ b/pkg/api/flamenco-manager.yaml @@ -340,11 +340,11 @@ paths: description: Size of the file in bytes. responses: "200": - description: The file is known to the server. - "420": - description: The file is currently being uploaded to the server. - "404": - description: The file does not exist on the server. + description: Normal response. + content: + application/json: + schema: + $ref: '#/components/schemas/ShamanFileStatus' default: description: unexpected error content: @@ -402,10 +402,8 @@ paths: `X-Shaman-Can-Defer-Upload: true` was sent in the request. "409": description: Checkout already exists. - content: - application/json: - schema: - $ref: '#/components/schemas/Error' + "417": + description: There was a mismatch between the request parameters and the actual file size or checksum of the uploaded file. default: description: unexpected error content: @@ -722,6 +720,15 @@ components: s: 127 p: logging.go + ShamanFileStatus: + type: object + description: Status of a file in the Shaman storage. + properties: + "status": + type: string + enum: [unknown, uploading, stored] + required: [status] + securitySchemes: worker_auth: description: Username is the worker ID, password is the secret given at worker registration. diff --git a/pkg/api/openapi_client.gen.go b/pkg/api/openapi_client.gen.go index 6bf2ced7..d59dc9d5 100644 --- a/pkg/api/openapi_client.gen.go +++ b/pkg/api/openapi_client.gen.go @@ -1383,6 +1383,7 @@ func (r ShamanCheckoutRequirementsResponse) StatusCode() int { type ShamanFileStoreCheckResponse struct { Body []byte HTTPResponse *http.Response + JSON200 *ShamanFileStatus JSONDefault *Error } @@ -2024,6 +2025,13 @@ func ParseShamanFileStoreCheckResponse(rsp *http.Response) (*ShamanFileStoreChec } switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest ShamanFileStatus + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { diff --git a/pkg/api/openapi_spec.gen.go b/pkg/api/openapi_spec.gen.go index 2682fbdf..8c665578 100644 --- a/pkg/api/openapi_spec.gen.go +++ b/pkg/api/openapi_spec.gen.go @@ -18,85 +18,86 @@ import ( // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+w8227cRpa/UuAskAmWfdHFkq2n1dhxIiOJhUieLBAbcpE83V1WsYqpKqrdMQTMR+yf", - "7A6wDztP+wOeP1qcupDFJltqJZLj2d08BC2yLud+pz8kuSwrKUAYnRx9SHS+gJLan8das7mA4pzqS/y7", - "AJ0rVhkmRXLUeUuYJpQY/EU1YQb/VpADu4KCZCtiFkB+lOoS1DhJk0rJCpRhYG/JZVlSUdjfzEBpf/yT", - "gllylPxh0gI38ZBNnroNyXWamFUFyVFClaIr/PudzHC3f6yNYmLun19UiknFzCpawISBOaiwwj0d2C5o", - "Ofzi5jO1oaa+FR2k35lbiRhRfbkZkLpmBb6YSVVSkxy5B+n6wus0UfBzzRQUydFPYRESx+PSwBahsEal", - "iCQxVGnLrzfNvTJ7B7lBAI+vKOM04/BCZmdgDILTk5wzJuYciHbviZwRSl7IjOBpekBAFpLl7mf3nB8X", - "IMicXYFICWclM1bOrihnBf6/Bk2MxGcaiD9kTF4KviK1RhjJkpkFcUSzl+PdjQj2iL8ubAXMaM1NH67z", - "BRD/0sFB9EIuhQeG1BoUWSLsBRhQJRP2/gXTgSRjd3x05vAVzZOJkZIbVvmLmGgvQnlUM5qDPRQKZhB1", - "d6KHf0a5hrRPXLMAhUBTzuWS4NZ1QAmdGVyzAPJOZmRBNckABNF1VjJjoBiTH2XNC8LKiq9IARzcNs4J", - "vGfaHUj1pSYzqdzR72SWEioKNCCyrBjHNcyMX4tW0DMpOVBhMbqivE+f05VZSEHgfaVAayYt8TMguLqm", - "BgqkkVSFQzDwASwmXdY1cDW8SfuicQmrPgwnBQjDZgyUP6QR+ZSUtTYITy3Yz7UTRM+0d14RBu9BxaBq", - "PqALx2JF4L1RlFA1r0u0MEHesmo1xo16fCZLOHW6tfrjlyRHNtQaClyZK6AGHKpe/1YRDK2Kt5blDiLE", - "yhIKRg3wFVGARxFqUS1gxgTDDSkaAns9XplamsjaeIioMiyvOVUNHzbIg66zYD5vsroDhurM72xU/c4n", - "nPvtV0yzdSUzqr6JQKi4XdXy8vDqxBlIJFZQK0X+yNklEEr+xEGgENOiGEnx5ZicgcHj3lqGvHVmxvlj", - "KpwtEJQ3d5gFNXh1zQvxhRXIxlKBKKwB0cOEXnMxqAB+0ZZu4azl05p3qLMRvnHi4BQi8Jw8rZUCYfiK", - "SLTjNJxrNSyy5HpM3n5zfPbNV88unp98+9XF6fH5N29dlFIwBbmRakUqahbkn8nb18nkD/a/18lbQqsK", - "SVo4tEHUJeI3YxwucH2SJgVT4ad97D3qguoFFBftyjcDCrxJaPoG3lMgwj6yGs59UU1OngV9tmij0HiR", - "GJPvJRGg0dZpo+rc1Ao0+aN1XzolBcvxKqoY6C8JVUB0XVVSmXXUPfApRjZ7u4g0l9QkqZWFW5Ecxi54", - "+/ZOFyUyTb6jgs5BORfAjFV9WqKBHggNOM2A3y1k88TcPtwcCml60cCaOniRcOBFd96mG0itAeP+LdMm", - "CIOV7s1069MohHG/DuPzjkXcgG57xRCCIV7voeVfEAXopa3LokS74NBHmdYSvYe8NnBbHrE5SG8EKHod", - "wBtmXLRlCKOvlJIKD1vPZAroROdBY/qpQQla0/kQvGsA2TPb9UPQPOe0BJHLP4PSPljckjJX7Y6boQgL", - "vV4NQfHCpV6U85ez5OinmyXsLMSHuOs67RHSxiJDEoMvbDTHStCGlhXao0DughoY4Zuh0IkNHPfq1cmz", - "4GZe2OzolsRq25wOTUWT0tVVcc/YrHHHQhpo1t7XAPvm+o1j0HdgaEENtYwqCht2UX7aoX0P47U4U2XM", - "KKpWpPSHeberx+Q7qaziVhzexz4npwK9Vikx/rcWq0YtJ2/pOBvnb4mQxtEhhMmXYENPeE/xLC/QVtCO", - "krNKMQPkuWLzBXohjFHGUFLGEepVpkD8S+ZdoFTzsMLpQHJmF5Az89//dQU8MmwdQT6LfMQwnVw0N7i3", - "EZDgQGlu2JXNnKnIkQIuia44GP9bOGIxKUYzytyK5kdFMURP0uTnGmr7g6p8wa6in84/u+NHKBnW7ftD", - "Og/sb3dKjSQaxZcnabKkNskbzaQaYSSjBx38DzBn2oCCwhnjvsmhRYGJ16BAcarNhSVKt3ISOW+WX242", - "55waVJJh7y5nZknVBte/le46lFr1bVztRVMF6brSWwsFv6lq09AibYgaV28CMdIkd6GxhTJZp3JEmQ0Y", - "Ddn0M8hrxcxqg7/b2ond5L3OFrSk4ukC8ktZDxRTMKGRM2KF0RVszAKYImffHO8+OiA5btR1mRLNfrHx", - "b7YyoF34WIBGEAiXuTMwPqfK/W1tLrBmbhT8bH1Ynhwle4+y6f6TnXz3MJvu7e0VO7Ns/9Esnx4+fkJ3", - "dnM6Pch2ioP9abH76ODJ4eNp9nh6WMCj6X5xON19AlPkEprzJtEdzyUyITna2d/dR+eHtxzsZYe7+cFe", - "9mR/d39W7OxlT/YOp7PsYDo9eDJ9PM336M6jw53DfLZHi/393YO9R9nO48P8gD5+8mh6+MTfwuV8jolS", - "c8XuoTX+XcZZ9CIJXvO+A0zoEjv4TGTKYJmif8JzxiO6+/Spr6EbNnZ4OxBQrQdNFnmEY0jibgxlkTSb", - "xfQHt7IMxfJfJapWNDsYPaz03b+k/W6Cda/ycX+yEUezg7luq/xtaRRTmxC6rwlAGcVpDxf5+Bd7H/+N", - "/P0vH//68W8f/+PjX//+l4//+fFvH/897gwcPZp282Z/y0VeFslR8sH/eY1OaFGLywtkR3K0hzgZRXNz", - "QeuCyRA1IXOtAThKJsrunOjZ5J3M0AeDgJ3dvbE9Mo6GT7//Gv+sdHK0u58mM8zPdHKU7Ix2UMZZSeeg", - "L6S6uGIFSOS9fZKkiaxNVRtXl4H3BoRLeZJxZaMmB8GFW9UFyV3SABXJh2bIqpFHfOS2JD11iPl4S7rQ", - "hObbtpuawiIyZ6D3FLHrtkwlLI0Knzf7cx+P+IZQA9WQbkTdrTuExE3w20SrGL60wfFAqOvD5KFwFWF4", - "ZZOigTpb847Y2qswmJ9QX2VAHXXplCvfW0TI63o63T0gXM61q83axiczX2hfq/BtgrWQOIp4uzC8FDDi", - "TPhKuShYjhcuFxRPzJuK58KWJjFxCr7FXjwmL69ALdE2aFIpuGKy1nzlcAmXNknakFXlcqBJ962cEwQq", - "6szgbSlZMs4xnQuFUgTaksJeCFRx5soz/bi4Iwvb9kSHcjTHHZeGKGqGqx6/PomAXIEZfvUbk4E1RfI3", - "deL4wSuiPODNRnqcsbl4eVdKhLzgYnMx6N7RjnKaDdj2oLoBa0MNPF1QMYc+6k5jL1pDcafkb51b64dt", - "BVSxCap7gOUWCLpGVxuqjCsV0CW9tBml5gAVBh82w0sTvahNIZe24wPar5azGVqCAdvqlMXmiGcItUNv", - "aQG4oDX6+F7NTYNC3qO5RRPmFpOTZympqNZLqYrwymmH6/ATasJSFak92hlLL9ucoprlreFZGFMl1wgj", - "EzPpCrTC0Ny0NdGmdkrOgaLy1Yr7nfpoMpmF8IzJSb8U9oNrvT2nqiSlq76T49MTzL1ZDkJDdM/Xp99e", - "7fXOXy6X47moMVqb+D16Mq/4aG88HYMYL0zpalTM8A60/rokKuEmO+PpeIqrZQWCVgxDO/sIfaNZWM5M", - "aMVspGVlUmpLCpRMS8yTwrXfSmZcNdRL+p9ksQrkA2H30KrizCXVk3faWQ0nt7dJdbf0e92jqm0NSR8m", - "J7HQY/RotUBXEimFN+1Op/cG2Q0ALakmus5z0LOa8xVxgwl2isC77CtW1JS7WYbx2nTIvUDnajAD8NkX", - "JJRYrErWZUnVqmEmoUTA0naP0Jc3UuRbRlGPxbptilGjbero5E3nuBehB+1GKkAUlWTCWHwb0Zo03mEO", - "A/L1NZim0fWAzOx31QZI1yxqO2trBPwaDOG97pttTNnUvtucvIF07VUN+d+1I08d+n14J7MLVlxvJOFz", - "MPnCaWjc2/rpQ8IQK19c8ZbHHdZTpDSi4211yTe/j9JZq91lh8XcviA0c8MhlndbyK3bJApvO0uEPJA9", - "Cn02yeyfmw7Yg5FivY83QBaBnOIkgDAgrEiQRsI8Xs1gzXeN2wjEwgx1jVgufHBdm1r7iSAjXY3G/cU0", - "JhY1RVNI2+s0qCsM/QNZnb+eKN8tGC3bZsGg6wltBd9UeBj/M5A6DBC6Tf8C9J/UFfUaLNvIwif0ObWA", - "9xXkBgoCfk0sQgF873iWgZ9B6vyDNwObHEtQYtudel2iNJuLkZzNbohiMBWazfrqut+PSD8/QvqQ2pr0", - "TjD90xs0xi3NvqPqMo6iqSYhWL+F2k8p973YoO+YxnsDEgKDS2GH0mD1hQIyl25Y1x4/HmaJuIUj4kGV", - "2l+xWZ2betyn1OV+lvoPocxby+BxbRYgjCta+dIYSkNoti2beZ17FkgFtFjhKjzPNfw65TrWMrwvrsZX", - "Awf9fcSy5PeWDAspye170pYertNNxoxs3vF5i9TdxcOFJMswRbsABW7SdbWBCMNyMMqjQs2g8Roo6jyo", - "IYsvGiDv941rdHhuYc/+d/k9b8893xwRxuQcY9Pcfm+Q2elYmqPB4FC4eN8V670taZsHHVlJiVRouQJV", - "gn0BNeIyp9yaNsr1fduzK+hgU+ueqBr/FdYG95ovoKg5nLvpj4fLq+NvwgYYa78GiwsKmwzV99J/+NGd", - "4bb5RRjxvE6T/ene/ZWeOuMsA8Cfggq1jWcgmDOa+9MnA58eOQFkmghpgqdzXS0nTinRMry2389AZ5bV", - "oW47uUTIpUN1d+/TupagRVQglDIzlAkbdlvoUpLVxo2cz6X9DEhIa2edtt1RY1+602lzfkSN21TJypT2", - "Aq4Gyk6Rhkw+2D6CL58M60rUD9ymguIP/O0llPt3FxEmm3TRx0NMOBBDDePO3uJ8AeGspTWtOVTBow6q", - "yLnvT1qP7K1GLEaOaVZPTPdsqzPx+f8obulV2yp2vVKzqlhuyyRxZ7dScq5A69QPy/qvnxSZUcZrBbf6", - "luBRNIiiUw1DcofT0YphROTURNv5pUmYwJq4ceHJh/Dg5NkNCrM2o7eN0rTn3qg3n0hP1hAY4HxndqsJ", - "LWVtxr9OXcJdVqD9bHbcQoj15mFluoGEcpcx2Q8v9fhzKhg9dR8fRh9nuXxOr0rOhCtxBCZ4HrnOi8Go", - "D/wj66so52RBr8B9iFpXXFI/pOCEgGQws3PqlPPmc9bWpbSa5zRmTfPOPECU6FhgLDCdyVKqgA5rnlqb", - "IdxG5Tpzhw+pIp2LtlWTT1pO2RLOOvMMQq4giaEgMeXTYI6dDACpBQbjwqH2eSmHHY4kNAhwjLsFN3wu", - "XUllNFkuWL7wHKKqQexW0T7GKBWvyTnDZCnKr7sHtgG7FL5Pr67CF3tusRR+rTaM8xaESB/sed7/6Lq8", - "nnywT9gvcO20AYmjNynGc8bhzEgFT70ArnmkradM7Sf/G9yXrss7Oa+0/w8d/ALxZZ2B34FbAwW2ubWd", - "bd3QnutHYw4ETZyge4sVNW32N8V9dmMhwYVm1n90ee/zl1vubUfZMkC1i43z2lmfle7Fs3+2hGAR8gTw", - "Sulhd99K99XsJiPfyPL/bTFOh0TH26IQOmv3xToztjtZwAyUl6LGlVtq2KDgdbI7ffw6aUs5dnLRprrC", - "yiBRYGqFaYn9TN5EohpiJTfu0VGYDsNdkky5lu4MLUuQAghwvSbyQ2BaabEEXAAtbIvMk/BfR+6a0VMq", - "Rs8Qz9Ere0AyQMPoo/whGkrF5kxQbu/E88fkZObHQ7mMx0l9XJEigcOYJxM5rwv3D2a0SmonPlNv7ZEX", - "lNkVBWS1+wJlC9xeesBGzz1gyb2kBjI3YEbaKKB2vrMZrE9eYIBowzS306mzWHkVGsigvUh8oZstD5kP", - "7E4f32BBF7SVy05wOybfS5t2Uv+vZViGoExm4Pjs5dvLXVcwPV8rJXPQliJd6+z8+tuNEnlEkAhv3YyU", - "U9ZYmFAQ/j/ViTIItPS+DrbZj3Q4uvbSsncmVc4yviI5l9qVHL45Pz9FMRVgv+Z0TA/VFm9IZ0wwvQDd", - "MUdA4D3NDdG0BB9QGmlHyXFLIWuM9dwGvdHDxVUQXNnmCH3sfFkFfzs/58YiJ0nUBur9azrdoZ/eEBsz", - "Gvhs3JoZO9rSt4gvZBa6lLZc8nMNioFOo8G2dG1OaNyZptIDhx6fnnRH6+ImlSzLWviZfrS066BHx/tq", - "z4ALdvQ7Pj1J7UVWcloeeoSsXcG/38msSUV1dL7n1/Wb6/8JAAD//+OqSK73TQAA", + "H4sIAAAAAAAC/+w86W4cR3qvUugN4DXSc/AQKfFXuJJlU7AtwqTWASyCqu7+ZqbE6qp2VTVHY4HAPkTe", + "JFkgP7K/8gLaNwq+OvqYriFHNilrk/iHMeyu47vv1vskl2UlBQijk6P3ic4XUFL781hrNhdQnFN9hX8X", + "oHPFKsOkSI56bwnThBKDv6gmzODfCnJg11CQbEXMAsiPUl2BGidpUilZgTIM7C25LEsqCvubGSjtj39S", + "MEuOkj9MWuAmHrLJU7chuUkTs6ogOUqoUnSFf7+VGe72j7VRTMz988tKMamYWXUWMGFgDiqscE8j2wUt", + "4y9uP1Mbauo70UH6nbmViBHVV5sBqWtW4IuZVCU1yZF7kK4vvEkTBT/XTEGRHP0UFiFxPC4NbB0U1qjU", + "IUkXqrTl10Vzr8zeQm4QwONryjjNOLyQ2RkYg+AMJOeMiTkHot17ImeEkhcyI3iajgjIQrLc/eyf8+MC", + "BJmzaxAp4axkxsrZNeWswP/XoImR+EwD8YeMyUvBV6TWCCNZMrMgjmj2cry7EcEB8deFrYAZrbkZwnW+", + "AOJfOjiIXsil8MCQWoMiS4S9AAOqZMLev2A6kGTsju+cGb+ieTIxUnLDKn8RE+1FKI9qRnOwh0LBDKLu", + "TvTwzyjXkA6JaxagEGjKuVwS3LoOKKEzg2sWQN7KjCyoJhmAILrOSmYMFGPyo6x5QVhZ8RUpgIPbxjmB", + "d0y7A6m+0mQmlTv6rcxSQkWBBkSWFeO4hpnxa9EKeiYlByosRteUD+lzujILKQi8qxRozaQlfgYEV9fU", + "QIE0kqpwCAY+gMWkz7oGroY36VA0rmA1hOGkAGHYjIHyhzQin5Ky1gbhqQX7uXaC6Jn21itC9B5UDKrm", + "EV04FisC74yihKp5XaKFCfKWVasxbtTjM1nCqdOt1R+/JDmyodZQ4MpcATXgUPX6t+rA0Kp4a1k+QoRY", + "WULBqAG+IgrwKEItqgXMmGC4IUVDYK/HK1NLE1kbDxFVhuU1p6rhwwZ50HUWzOdtVjdiqM78zkbVP/qE", + "c7/9mmm2rmRG1bcRCBW3r1peHl6dOAOJxApqpcgfObsCQsmfOAgUYloUIym+HJMzMHjcG8uQN87MOH9M", + "hbMFgvLmDrOgBq+ueSG+sALZWCoQhTUgOk7oNReDCuAXbekWzlo+rXmHOhvhGycOTiECz8nTWikQhq+I", + "RDtOw7lWwzqWXI/Jm2+Oz7756tnl85Nvv7o8PT7/5o2LUgqmIDdSrUhFzYL8M3nzOpn8wf73OnlDaFUh", + "SQuHNoi6RPxmjMMlrk/SpGAq/LSPvUddUL2A4rJdeRFR4E1CMzTwngId7DtWw7kvqsnJs6DPFm0UGi8S", + "Y/K9JAI02jptVJ2bWoEmf7TuS6ekYDleRRUD/SWhCoiuq0oqs466Bz7FyGZvF5HmkpoktbJwJ5Jx7IK3", + "b+90USLT5Dsq6ByUcwHMWNWnJRroSGjAaQb840I2T8ztw81YSDOIBtbUwYuEA69z5126gdSKGPdvmTZB", + "GKx0b6bbkEYhjPt1GJ/3LOIGdNsrYgiGeH2Aln9BFKCXti6LEu2CQx9lWkv0DvLawF15xOYgvRGgzusA", + "XpxxnS0xjL5SSio8bD2TKaAXnQeNGaYGJWhN5zF41wCyZ7brY9A857QEkcs/g9I+WNySMtftjtuhCAu9", + "XsWgeOFSL8r5y1ly9NPtEnYW4kPcdZMOCGljkZjE4AsbzbEStKFlhfYokLugBkb4JhY6schxr16dPAtu", + "5oXNju5IrLbN6dBUNCldXRX3jM0adyykgWbtfQ2wFzcXjkHfgaEFNdQyqihs2EX5aY/2A4zX4kyVMaOo", + "WpHSH+bdrh6T76SyiltxeNf1OTkV6LVKifG/tVg1ajl5Q8fZOH9DhDSODiFMvgIbesI7imd5gbaCdpSc", + "VYoZIM8Vmy/QC2GMMoaSMo5QrzIF4l8y7wKlmocVTgeSM7uAnJn//q9r4B3D1hPks46PiNPJRXPRvY2A", + "BAdKc8OubeZMRY4UcEl0xcH438IRi0kxmlHmVjQ/KoohepImP9dQ2x9U5Qt23fnp/LM7foSSYd2+P6T3", + "wP52p9RIolH38iRNltQmeaOZVCOMZHTUwf8Ac6YNKCicMR6aHFoUmHhFBYpTbS4tUfqVk47zZvnVZnPO", + "qUEliXt3OTNLqja4/q1016HUqm/jai+bKkjfld5ZKPhNVZuGFmlD1G71JhAjTXIXGlsok3UqdyizAaOY", + "TT+DvFbMrDb4u62d2G3e62xBSyqeLiC/knWkmIIJjZwRK4yuYGMWwBQ5++Z499EByXGjrsuUaPaLjX+z", + "lQHtwscCNIJAuMydgfE5Ve5va3OBNXOj4Gfrw/LkKNl7lE33n+zku4fZdG9vr9iZZfuPZvn08PETurOb", + "0+lBtlMc7E+L3UcHTw4fT7PH08MCHk33i8Pp7hOYIpfQnDeJ7ngukQnJ0c7+7j46P7zlYC873M0P9rIn", + "+7v7s2JnL3uydzidZQfT6cGT6eNpvkd3Hh3uHOazPVrs7+8e7D3Kdh4f5gf08ZNH08Mn/hYu53NMlJor", + "dg+t8e8zzqLXkeA17xthQp/YwWciU6JliuEJzxnv0N2nT0MN3bCxx9tIQLUeNFnkEY6YxN0ayiJpNosp", + "AtMa+DUa2ecuWUfCBGFzO4k2UtE5DONWPXAYtbgScimsK+eSFs684wFQROzxGgbB6W9E4ge3uAwV/1+l", + "b1a/emx5WBW6f3X53bTjXoX8/gS8G5JHE/bWgrX1XczPQv6xJgBlJ9h8uPDNv9j78G/k73/58NcPf/vw", + "Hx/++ve/fPjPD3/78O/d9sbRo2k/+fe3XOZlkRwl7/2fN+hJF7W4ukR2JEd7iJNRNDeXtC6YDKEfMtda", + "saNkouzOiZ5N3soMAwkQsLO7N7ZHdkP60++/xj8rnRzt7qfJDJNMnRwlO6MdlHFW0jnoS6kur1kBEnlv", + "nyRpImtT1cYVl+CdAeHytmRcWdvgILh0q/oguUsaoDryoRmyauQRH7ktyUAduny8I+dp8otte2ZNdRSZ", + "E2mgddh1V7oVlnaqt7fbSR9U+a5WA1VMNzotuo+I65sIvgm5MQZrI/xIvO5j/VjMjTC8spldpFjYvCO2", + "gCwMJlnUl0pQR11O6HoQzk+9rqfT3QPC5dz7LNu9ZeYL7QsuvtexFtd3wvY+DC8FjDgTvtwvCpbjhcsF", + "xRPzpmy7sPVVzP6Cb7EXj8nLa1BLtA2aVAqumaw1XzlcwqVNphmzqlxGOo3fyjlBoDrtJbwtJUvGOeak", + "odqLQFtS2AuBKs5cjWkY3PdkYdvGbizRdNxxuZSiJl66+fWZEOQKTPzVb8xo1gMOd1MvGYle0UlmLjbS", + "44zNxcuPpURIbi43V7TuHe1OYrYB2wFUt2BtqIGnCyrmMETdaexlayg+KoONhoedw7YCqtgE1T3AcgcE", + "faOrDVXGBcR0Sa9sWqw5QIXBh01T00QvalO4ANqA9qvlbIaWIB4/a5/oniHUDr2lBeCS1ujjB4VDDQp5", + "j+YWTZhbTE6epaSiWi+lKsIrpx1uTIFQE5aqjtqjnbH0sh02qlneGp6FMVVygzAyMZOuyiwMzU1b2G0K", + "wOQcKCpfrbjfqY8mk1kIz5icDOt5P7j+4XOqSlK6FgI5Pj1J0oSzHISGzj1fn357vTc4f7lcjueixmht", + "4vfoybzio73xdAxivDClK7Qxw3vQ+uuSTh062RlPx1NcLSsQtGIY2tlH6BvNwnJmQitmIy0rk1JbUqBk", + "WmKeFK6HWDLjSrpe0v8ki1UgHwi7h1YVZ64yMHmrndVwcnuXVPfr1zcDqtr+lvRhctIVeowerRboSiKl", + "8Kbd6fTeILsFoCXVRNd5DnpWc74ibrrCjkJ4l33NippyN5AxXhtxuRfoXCEpAp99QUKdyKpkXZZUrRpm", + "EkoELG0LDH15I0W+79VpFFm3TTFqtJ0pnVz0jnsRGuluLgREUUkmjMW3Ea1J4x3mEJGvr8E03boHZOaw", + "NRghXbOobQ+uEfBrMIQPWoi2u2ZT+36H9RbStVc15H/bzm316Pf+rcwuWXGzkYTPweQLp6HdBt1P7xOG", + "WPkKkbc87rCBIqUdOt5VXL34fZTOWu0+Oyzm9gWhmZtwsbzbQm7dJlF421ki5IHsndBnk8z+uWnjPRgp", + "1puREbII5BQnAYSIsCJBGgnzeDXTQd81biMQCzPUNWK58MG1nmrtx5qMdDUa9xfTmFjUFE0hba/ToK4x", + "9A9kdf56onzLY7RsOx5R1xN6I74z8jD+J5I6RAjdpn8B+k/qigZdom1k4RP6nFrAuwpyAwUBv6YrQgF8", + "73iWgZ9B6vyDi8gmxxKU2HanXpcozeZiJGezW6IYTIVms6G67g8j0s+PkD6ktia9F0z/dIHGuKXZd1Rd", + "daNoqkkI1u+g9lPKfUM56Dum8d6AhMDgStjJOlh9oYDMpZs4tseP4ywRd3BEPKhS+ys2q3NTj/uUujzM", + "Uv8hlHlrGTyuzQKEcUUrXxpDaQhNnGUzdHTPAqmAFitchee5rmWvXMdahg/F1fhqYNTfd1iW/N6SYSEl", + "uX1P2tLDTbrJmJHNOz5vkfp48XAhyTKMAi9AgRvXXW0gQlwORnmnUBM1XpGizoMasu5FEfJ+37hGh+cW", + "9ux/l9/z9tzzzRFhTM4xNs3tRxOZHfGlORoMDoWL912x3tuStnnQk5WUSIWWK1Al2BdQIy5zyq1po1zf", + "tz27hh42tR6IqvGfkm1wr/kCiprDuRthebi8uvthW4Sx9pO2bkFhk6H6XvqvV/qD6Da/CHOqN2myP927", + "v9JTbyYnAvwpqFDbeAaCOaO5P30S+X7KCSDTREgTPJ3rajlxSomW4bX9CAh6A7kOddvJJUIuHaq7e5/W", + "tQQtogKhlJmhTNiw20KXkqw2bm5+Lu23TEJaO+u07SM19qU7nTbnd6hxlypZmdJewFWk7NTRkMl720fw", + "5ZO4rnT6gdtUUPyBv72Ecv/uooPJJl308RATDsRQw/hob3G+gHDW0prWHKrgUaMqcu77k9Yje6vRFSPH", + "NKsnpn+21Znu+f8obulV2yp2vVKzqlhuyyTdzm6l5FyB1qmf+PWfcCkyo4zXCu70LcGjaBBFrxqG5A6n", + "oxXDiMipibbzS5MwRjZxM8+T9+HBybNbFGZt0HAbpWnPvVVvPpGerCEQ4XxvdqsJLWVtxr9OXcJdVqD9", + "gHm3hdDVm4eV6QYSyl3GZL8e1ePPqWD01H1B2fnCzOVzelVyJlyJIzDB88h1XgxGfeAfWV9FOScLeg3u", + "a1o3BegMj58mzGBmh+0p5803ua1LaTXPacya5p15gCjRXYGxwPTGY6kCGtc8tTZDuI3K9eYOH1JFehdt", + "qyaftJyyJZx15hmEXEESQ0G6lE+DOXYyAMRPjjrUPi/lsMORhAYB7uJuwQ3ffFdSGU2WC5YvPIeoahC7", + "U7SPMUrFa3LOMFnq5Nf9A9uAXQrfp1fX4bNDt1gKv1YbxnkLQkcf7Hne/+i6vJm8t0/YL3DjtAGJozcp", + "hpsmlgqeegFc80hbT5naf7dgg/vSdflRzisd/msNv0D3st7Ab+TWQIFtbm1nWy8eXNs6s9uxakS/jvD5", + "6U53dq+dMZf9GXPXu3IfbA/V5DYj3cji/20xTGMJgLclIfTV7rN5Zmx3sYAZKNIM6jtXbKlhnfrrZHf6", + "+HXSlmLs5KFNVQVfkQxDAlMrTCvst/oterqJddy4hrPs3kX3GO6SXMq1dGdoWYIUQIBre047fRkD00qL", + "JeACaGFbXJ6E/zpy14yeUjF6hniOXtkDkggNO/8yQIyGUrE5E5TbO/H8MTmZ+fFOLrvjoD4uSJHAYUyT", + "iZzXhftXO1pjbSc2U2+tkReU2RUFZLX7DGYL3F56wEbPPWDJvYT2MjdgRtoooHY+sxmMT15ggGfDLLfT", + "qbNYeRWKZMBeJL7QzZaHjOd3p4/jebAVywVt5bIXnI7J99KmjdT/kx2WISiTGTg+e/n2ctcXTM/XSskc", + "tKVIBiim4XTnl99slMgjgkR442acnLJ2hQkF4f9TlU4GgJbe17E2+5EeR9deWvbOpMpZxlck51K7ksE3", + "5+enKKYC7CeljumhWuIN6YwJphege+YICLyjuSGaluADQiPtKDhuKWSNsZrboDd6uG4VA1e2Mf4QO18W", + "wd/Oz7mxxknSaeMM/kmf/tDOYAiNGQ18Nm7NjB1NGVrEFzILXUZb7vi5BsVAp53BtHRtzmfcm4bSkUOP", + "T0/6o3HdJpMsy1r4mXy0tOugd4731ZqIC3b0Oz49Se1FVnJaHnqErF3Bv9/KrEklded8z6+bi5v/CQAA", + "//+lKaE0fE4AAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/pkg/api/openapi_types.gen.go b/pkg/api/openapi_types.gen.go index d4581124..4e356b63 100644 --- a/pkg/api/openapi_types.gen.go +++ b/pkg/api/openapi_types.gen.go @@ -66,6 +66,15 @@ const ( JobStatusWaitingForFiles JobStatus = "waiting-for-files" ) +// Defines values for ShamanFileStatusStatus. +const ( + ShamanFileStatusStatusStored ShamanFileStatusStatus = "stored" + + ShamanFileStatusStatusUnknown ShamanFileStatusStatus = "unknown" + + ShamanFileStatusStatusUploading ShamanFileStatusStatus = "uploading" +) + // Defines values for TaskStatus. const ( TaskStatusActive TaskStatus = "active" @@ -247,6 +256,14 @@ type ShamanCheckout struct { } `json:"req"` } +// Status of a file in the Shaman storage. +type ShamanFileStatus struct { + Status ShamanFileStatusStatus `json:"status"` +} + +// ShamanFileStatusStatus defines model for ShamanFileStatus.Status. +type ShamanFileStatusStatus string + // Set of files with their SHA256 checksum and size in bytes. type ShamanRequirements struct { Req []struct { diff --git a/pkg/shaman/cleanup.go b/pkg/shaman/cleanup.go index cb35974c..1baa78f8 100644 --- a/pkg/shaman/cleanup.go +++ b/pkg/shaman/cleanup.go @@ -46,7 +46,7 @@ type GCStats struct { } func (s *Server) periodicCleanup() { - defer packageLogger.Debug("shutting down period cleanup") + defer log.Debug().Msg("shaman: shutting down period cleanup") defer s.wg.Done() for { diff --git a/pkg/shaman/fileserver/checkfile.go b/pkg/shaman/fileserver/checkfile.go index 9baf13d4..f70dca4c 100644 --- a/pkg/shaman/fileserver/checkfile.go +++ b/pkg/shaman/fileserver/checkfile.go @@ -23,24 +23,10 @@ package fileserver import ( - "context" - "net/http" - "git.blender.org/flamenco/pkg/shaman/filestore" ) -var responseForStatus = map[filestore.FileStatus]int{ - filestore.StatusUploading: 420, // Enhance Your Calm - filestore.StatusStored: http.StatusOK, - filestore.StatusDoesNotExist: http.StatusNotFound, -} - -func (fs *FileServer) checkFile(ctx context.Context, w http.ResponseWriter, checksum string, filesize int64) { +func (fs *FileServer) CheckFile(checksum string, filesize int64) filestore.FileStatus { _, status := fs.fileStore.ResolveFile(checksum, filesize, filestore.ResolveEverything) - code, ok := responseForStatus[status] - if !ok { - packageLogger.WithField("fileStoreStatus", status).Error("no HTTP status code implemented") - code = http.StatusInternalServerError - } - w.WriteHeader(code) + return status } diff --git a/pkg/shaman/fileserver/logging.go b/pkg/shaman/fileserver/logging.go deleted file mode 100644 index ed34ba93..00000000 --- a/pkg/shaman/fileserver/logging.go +++ /dev/null @@ -1,29 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package fileserver - -import ( - "github.com/sirupsen/logrus" -) - -var packageLogger = logrus.WithField("package", "shaman/receiver") diff --git a/pkg/shaman/fileserver/receivefile.go b/pkg/shaman/fileserver/receivefile.go index 15b77053..bdd08524 100644 --- a/pkg/shaman/fileserver/receivefile.go +++ b/pkg/shaman/fileserver/receivefile.go @@ -24,153 +24,156 @@ package fileserver import ( "context" + "errors" "fmt" "io" - "net/http" "git.blender.org/flamenco/pkg/shaman/filestore" "git.blender.org/flamenco/pkg/shaman/hasher" - "git.blender.org/flamenco/pkg/shaman/httpserver" - "git.blender.org/flamenco/pkg/shaman/jwtauth" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog" ) -// receiveFile streams a file from a HTTP request to disk. -func (fs *FileServer) receiveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, checksum string, filesize int64) { - logger := packageLogger.WithFields(jwtauth.RequestLogFields(r)) +// ErrFileAlreadyExists indicates that a file already exists in the Shaman +// storage. It can also be returned during upload, when someone else succesfully +// uploaded the same file at the same time. +var ErrFileAlreadyExists = errors.New("uploaded file already exists") - bodyReader, err := httpserver.DecompressedReader(r) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } +type ErrFileSizeMismatch struct { + DeclaredSize int64 + ActualSize int64 +} + +func (e ErrFileSizeMismatch) Error() string { + return fmt.Sprintf("file size mismatched, declared %d but received %d bytes", e.DeclaredSize, e.ActualSize) +} + +type ErrFileChecksumMismatch struct { + DeclaredChecksum string + ActualChecksum string +} + +func (e ErrFileChecksumMismatch) Error() string { + return fmt.Sprintf("file SHA256 mismatched, declared %s but received %s", e.DeclaredChecksum, e.ActualChecksum) +} + +// ReceiveFile streams a file from a HTTP request to disk. +func (fs *FileServer) ReceiveFile( + ctx context.Context, bodyReader io.ReadCloser, + checksum string, filesize int64, canDefer bool, +) error { + logger := *zerolog.Ctx(ctx) defer bodyReader.Close() - originalFilename := r.Header.Get("X-Shaman-Original-Filename") - if originalFilename == "" { - originalFilename = "-not specified-" - } - logger = logger.WithField("originalFilename", originalFilename) - localPath, status := fs.fileStore.ResolveFile(checksum, filesize, filestore.ResolveEverything) - logger = logger.WithField("path", localPath) - if status == filestore.StatusStored { - logger.Info("uploaded file already exists") - w.Header().Set("Location", r.RequestURI) - http.Error(w, "File already stored", http.StatusAlreadyReported) - return + logger = logger.With().Str("path", localPath).Logger() + + switch status { + case filestore.StatusStored: + logger.Info().Msg("uploaded file already exists") + return ErrFileAlreadyExists + case filestore.StatusUploading: + if canDefer { + logger.Info().Msg("someone is uploading this file and client can defer") + return ErrFileAlreadyExists + } } - if status == filestore.StatusUploading && r.Header.Get("X-Shaman-Can-Defer-Upload") == "true" { - logger.Info("someone is uploading this file and client can defer") - http.Error(w, "File being uploaded, please defer", http.StatusAlreadyReported) - return - } - logger.Info("receiving file") + logger.Info().Msg("receiving file") streamTo, err := fs.fileStore.OpenForUpload(checksum, filesize) if err != nil { - logger.WithError(err).Error("unable to open file for writing uploaded data") - http.Error(w, "Unable to open file", http.StatusInternalServerError) - return + return fmt.Errorf("opening file for writing uploaded data: %w", err) } - // clean up temporary file if it still exists at function exit. + // Clean up temporary file if it still exists at function exit. defer func() { streamTo.Close() fs.fileStore.RemoveUploadedFile(streamTo.Name()) }() - // Abort this uploadwhen the file has been finished by someone else. + // Abort this upload when the file has been finished by someone else. uploadDone := make(chan struct{}) uploadAlreadyCompleted := false defer close(uploadDone) receiverChannel := fs.receiveListenerFor(checksum, filesize) go func() { select { - case <-receiverChannel: case <-uploadDone: close(receiverChannel) return + case <-receiverChannel: } - - logger := logger.WithField("path", localPath) - logger.Info("file was completed during someone else's upload") + logger.Info().Msg("file was completed during someone else's upload") uploadAlreadyCompleted = true - err := r.Body.Close() + err := bodyReader.Close() if err != nil { - logger.WithError(err).Warning("error closing connection") + logger.Warn().Err(err).Msg("error closing connection") } }() + // TODO: pass context to hasher.Copy() written, actualChecksum, err := hasher.Copy(streamTo, bodyReader) if err != nil { if closeErr := streamTo.Close(); closeErr != nil { - logger.WithFields(logrus.Fields{ - logrus.ErrorKey: err, - "closeError": closeErr, - }).Error("error closing local file after other I/O error occured") + logger.Error(). + AnErr("copyError", err). + AnErr("closeError", closeErr). + Msg("error closing local file after other I/O error occured") } - logger = logger.WithError(err) - if uploadAlreadyCompleted { - logger.Debug("aborted upload") - w.Header().Set("Location", r.RequestURI) - http.Error(w, "File already stored", http.StatusAlreadyReported) - } else if err == io.ErrUnexpectedEOF { - logger.Info("unexpected EOF, client probably just disconnected") - } else { - logger.Warning("unable to copy request body to file") - http.Error(w, "I/O error", http.StatusInternalServerError) + logger = logger.With().Err(err).Logger() + switch { + case uploadAlreadyCompleted: + logger.Debug().Msg("aborted upload") + return ErrFileAlreadyExists + case err == io.ErrUnexpectedEOF: + logger.Debug().Msg("unexpected EOF, client probably just disconnected") + return err + default: + return fmt.Errorf("unable to copy request body to file: %w", err) } - return } if err := streamTo.Close(); err != nil { - logger.WithError(err).Warning("error closing local file") - http.Error(w, "I/O error", http.StatusInternalServerError) - return + return fmt.Errorf("closing local file: %w", err) } if written != filesize { - logger.WithFields(logrus.Fields{ - "declaredSize": filesize, - "actualSize": written, - }).Warning("mismatch between expected and actual size") - http.Error(w, - fmt.Sprintf("Received %d bytes but you promised %d", written, filesize), - http.StatusExpectationFailed) - return + logger.Warn(). + Int64("declaredSize", filesize). + Int64("actualSize", written). + Msg("mismatch between expected and actual size") + return ErrFileSizeMismatch{ + DeclaredSize: filesize, + ActualSize: written, + } } if actualChecksum != checksum { - logger.WithFields(logrus.Fields{ - "declaredChecksum": checksum, - "actualChecksum": actualChecksum, - }).Warning("mismatch between expected and actual checksum") - http.Error(w, - "Declared and actual checksums differ", - http.StatusExpectationFailed) - return + logger.Warn(). + Str("declaredChecksum", checksum). + Str("actualChecksum", actualChecksum). + Msg("mismatch between expected and actual checksum") + return ErrFileChecksumMismatch{ + DeclaredChecksum: checksum, + ActualChecksum: actualChecksum, + } } - logger.WithFields(logrus.Fields{ - "receivedBytes": written, - "checksum": actualChecksum, - "tempFile": streamTo.Name(), - }).Debug("File received") + logger.Debug(). + Int64("receivedBytes", written). + Str("checksum", actualChecksum). + Str("tempFile", streamTo.Name()). + Msg("File received") if err := fs.fileStore.MoveToStored(checksum, filesize, streamTo.Name()); err != nil { - logger.WithFields(logrus.Fields{ - "tempFile": streamTo.Name(), - logrus.ErrorKey: err, - }).Error("unable to move file from 'upload' to 'stored' storage") - http.Error(w, - "unable to move file from 'upload' to 'stored' storage", - http.StatusInternalServerError) - return + logger.Error(). + Err(err). + Str("tempFile", streamTo.Name()). + Msg("unable to move file from 'upload' to 'stored' storage") + return err } - http.Error(w, "", http.StatusNoContent) + return nil } diff --git a/pkg/shaman/fileserver/routes.go b/pkg/shaman/fileserver/routes.go deleted file mode 100644 index 3b5c0a16..00000000 --- a/pkg/shaman/fileserver/routes.go +++ /dev/null @@ -1,97 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package fileserver - -import ( - "errors" - "fmt" - "net/http" - "strconv" - - "git.blender.org/flamenco/pkg/shaman/jwtauth" - "github.com/gorilla/mux" - "github.com/sirupsen/logrus" -) - -// AddRoutes adds this package's routes to the Router. -func (fs *FileServer) AddRoutes(router *mux.Router, auther jwtauth.Authenticator) { - router.Handle("/files/{checksum}/{filesize}", auther.Wrap(fs)).Methods("GET", "POST", "OPTIONS") -} - -func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { - logger := packageLogger.WithFields(jwtauth.RequestLogFields(r)) - - checksum, filesize, err := parseRequestVars(w, r) - if err != nil { - logger.WithError(err).Warning("invalid request") - return - } - - logger = logger.WithFields(logrus.Fields{ - "checksum": checksum, - "filesize": filesize, - }) - - switch r.Method { - case http.MethodOptions: - logger.Info("checking file") - fs.checkFile(r.Context(), w, checksum, filesize) - case http.MethodGet: - // TODO: make optional or just delete: - logger.Info("serving file") - fs.serveFile(r.Context(), w, checksum, filesize) - case http.MethodPost: - fs.receiveFile(r.Context(), w, r, checksum, filesize) - default: - // This should never be reached due to the router options, but just in case. - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - } -} - -func parseRequestVars(w http.ResponseWriter, r *http.Request) (string, int64, error) { - vars := mux.Vars(r) - checksum, ok := vars["checksum"] - if !ok { - http.Error(w, "missing checksum", http.StatusBadRequest) - return "", 0, errors.New("missing checksum") - } - // Arbitrary minimum length, but we can fairly safely assume that all - // hashing methods used produce a hash of at least 32 characters. - if len(checksum) < 32 { - http.Error(w, "checksum suspiciously short", http.StatusBadRequest) - return "", 0, errors.New("checksum suspiciously short") - } - - filesizeStr, ok := vars["filesize"] - if !ok { - http.Error(w, "missing filesize", http.StatusBadRequest) - return "", 0, errors.New("missing filesize") - } - filesize, err := strconv.ParseInt(filesizeStr, 10, 64) - if err != nil { - http.Error(w, "invalid filesize", http.StatusBadRequest) - return "", 0, fmt.Errorf("invalid filesize: %v", err) - } - - return checksum, filesize, nil -} diff --git a/pkg/shaman/filestore/filestore.go b/pkg/shaman/filestore/filestore.go index 86745b5d..bff56a83 100644 --- a/pkg/shaman/filestore/filestore.go +++ b/pkg/shaman/filestore/filestore.go @@ -28,7 +28,7 @@ import ( "strconv" "git.blender.org/flamenco/pkg/shaman/config" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog/log" ) // Store represents the default Shaman file store. @@ -40,8 +40,8 @@ type Store struct { } // New returns a new file store. -func New(conf config.Config) Storage { - packageLogger.WithField("storageDir", conf.FileStorePath).Info("opening file store") +func New(conf config.Config) *Store { + log.Info().Str("storageDir", conf.FileStorePath).Msg("shaman: opening file store") store := &Store{ conf.FileStorePath, storageBin{conf.FileStorePath, "uploading", true, ".tmp"}, @@ -55,24 +55,16 @@ func New(conf config.Config) Storage { func (s *Store) createDirectoryStructure() { mkdir := func(subdir string) { path := path.Join(s.baseDir, subdir) - logger := packageLogger.WithField("path", path) - stat, err := os.Stat(path) - if err == nil { - if stat.IsDir() { - // Exists and is a directory; nothing to do. + logger := log.With().Str("path", path).Logger() + logger.Debug().Msg("shaman: creating directory") + + if err := os.MkdirAll(path, 0777); err != nil { + if os.IsExist(err) { + logger.Trace().Msg("shaman: directory exists") return } - logger.Fatal("path exists but is not a directory") - } - - if !os.IsNotExist(err) { - logger.WithError(err).Fatal("unable to stat directory") - } - - logger.Debug("creating directory") - if err := os.MkdirAll(path, 0777); err != nil { - logger.WithError(err).Fatal("unable to create directory") + logger.Error().Err(err).Msg("shaman: unable to create directory") } } @@ -100,27 +92,29 @@ func (s *Store) partialFilePath(checksum string, filesize int64) string { func (s *Store) ResolveFile(checksum string, filesize int64, storedOnly StoredOnly) (path string, status FileStatus) { partial := s.partialFilePath(checksum, filesize) - logger := packageLogger.WithFields(logrus.Fields{ - "checksum": checksum, - "filesize": filesize, - "partialPath": partial, - "storagePath": s.baseDir, - }) + logger := log.With(). + Str("checksum", checksum). + Int64("filesize", filesize). + Str("partialPath", partial). + Str("storagePath", s.baseDir). + Logger() if path = s.stored.resolve(partial); path != "" { - // logger.WithField("path", path).Debug("found stored file") + logger.Trace().Str("path", path).Msg("shaman: found stored file") return path, StatusStored } + if storedOnly != ResolveEverything { - // logger.Debug("file does not exist in 'stored' state") + logger.Trace().Msg("shaman: file does not exist in 'stored' state") return "", StatusDoesNotExist } if path = s.uploading.resolve(partial); path != "" { - logger.WithField("path", path).Debug("found currently uploading file") + logger.Debug().Str("path", path).Msg("shaman: found currently uploading file") return path, StatusUploading } - // logger.Debug("file does not exist") + + logger.Trace().Msg("shaman: file does not exist") return "", StatusDoesNotExist } @@ -145,11 +139,10 @@ func (s *Store) MoveToStored(checksum string, filesize int64, uploadedFilePath s if err := os.MkdirAll(targetDir, 0777); err != nil { return err } - logger := packageLogger.WithFields(logrus.Fields{ - "uploadedPath": uploadedFilePath, - "storagePath": targetPath, - }) - logger.Debug("moving uploaded file to storage") + log.Debug(). + Str("uploadedPath", uploadedFilePath). + Str("storagePath", targetPath). + Msg("shaman: moving uploaded file to storage") if err := os.Rename(uploadedFilePath, targetPath); err != nil { return err } @@ -161,7 +154,7 @@ func (s *Store) MoveToStored(checksum string, filesize int64, uploadedFilePath s func (s *Store) removeFile(filePath string) error { err := os.Remove(filePath) if err != nil { - packageLogger.WithError(err).Debug("unable to delete file; ignoring") + log.Debug().Err(err).Msg("shaman: unable to delete file; ignoring") } // Clean up directory structure, but ignore any errors (dirs may not be empty) @@ -177,8 +170,8 @@ func (s *Store) removeFile(filePath string) error { func (s *Store) RemoveUploadedFile(filePath string) { // Check that the file path is actually in the 'uploading' storage. if !s.uploading.contains("", filePath) { - packageLogger.WithField("file", filePath).Error( - "filestore.Store.RemoveUploadedFile called with file not in 'uploading' storage bin") + log.Error().Str("file", filePath). + Msg("shaman: RemoveUploadedFile called with file not in 'uploading' storage bin") return } s.removeFile(filePath) @@ -188,8 +181,8 @@ func (s *Store) RemoveUploadedFile(filePath string) { func (s *Store) RemoveStoredFile(filePath string) error { // Check that the file path is actually in the 'stored' storage. if !s.stored.contains("", filePath) { - packageLogger.WithField("file", filePath).Error( - "filestore.Store.RemoveStoredFile called with file not in 'stored' storage bin") + log.Error().Str("file", filePath). + Msg("shaman: RemoveStoredFile called with file not in 'stored' storage bin") return os.ErrNotExist } return s.removeFile(filePath) diff --git a/pkg/shaman/filestore/logging.go b/pkg/shaman/filestore/logging.go deleted file mode 100644 index bbbd1b7e..00000000 --- a/pkg/shaman/filestore/logging.go +++ /dev/null @@ -1,29 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package filestore - -import ( - "github.com/sirupsen/logrus" -) - -var packageLogger = logrus.WithField("package", "shaman/filestore") diff --git a/pkg/shaman/filestore/testing.go b/pkg/shaman/filestore/testing.go index f12b31d4..09e6420b 100644 --- a/pkg/shaman/filestore/testing.go +++ b/pkg/shaman/filestore/testing.go @@ -43,13 +43,7 @@ func CreateTestStore() *Store { conf := config.Config{ FileStorePath: tempDir, } - storage := New(conf) - store, ok := storage.(*Store) - if !ok { - panic("storage should be *Store") - } - - return store + return New(conf) } // CleanupTestStore deletes a store returned by CreateTestStore() diff --git a/pkg/shaman/httpserver/filefinder.go b/pkg/shaman/httpserver/filefinder.go deleted file mode 100644 index 680de25b..00000000 --- a/pkg/shaman/httpserver/filefinder.go +++ /dev/null @@ -1,68 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package httpserver - -import ( - "os" - "path/filepath" - - "github.com/kardianos/osext" - "github.com/sirupsen/logrus" -) - -// RootPath returns the filename prefix to find bundled files. -// Files are searched for relative to the current working directory as well as relative -// to the currently running executable. -func RootPath(fileToFind string) string { - logger := packageLogger.WithField("fileToFind", fileToFind) - - // Find as relative path, i.e. relative to CWD. - _, err := os.Stat(fileToFind) - if err == nil { - logger.Debug("found in current working directory") - return "" - } - - // Find relative to executable folder. - exedirname, err := osext.ExecutableFolder() - if err != nil { - logger.WithError(err).Error("unable to determine the executable's directory") - return "" - } - - if _, err := os.Stat(filepath.Join(exedirname, fileToFind)); os.IsNotExist(err) { - cwd, err := os.Getwd() - if err != nil { - logger.WithError(err).Error("unable to determine current working directory") - } - logger.WithFields(logrus.Fields{ - "cwd": cwd, - "exedirname": exedirname, - }).Error("unable to find file") - return "" - } - - // Append a slash so that we can later just concatenate strings. - logrus.WithField("exedirname", exedirname).Debug("found file") - return exedirname + string(os.PathSeparator) -} diff --git a/pkg/shaman/httpserver/gzip.go b/pkg/shaman/httpserver/gzip.go deleted file mode 100644 index a6653355..00000000 --- a/pkg/shaman/httpserver/gzip.go +++ /dev/null @@ -1,87 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package httpserver - -import ( - "bytes" - "compress/gzip" - "errors" - "io" - "net/http" -) - -// Errors returned by DecompressedReader -var ( - ErrContentEncodingNotSupported = errors.New("Content-Encoding not supported") -) - -// wrapperCloserReader is a ReadCloser that closes both a wrapper and the wrapped reader. -type wrapperCloserReader struct { - wrapped io.ReadCloser - wrapper io.ReadCloser -} - -func (cr *wrapperCloserReader) Close() error { - errWrapped := cr.wrapped.Close() - errWrapper := cr.wrapper.Close() - - if errWrapped != nil { - return errWrapped - } - return errWrapper -} - -func (cr *wrapperCloserReader) Read(p []byte) (n int, err error) { - return cr.wrapper.Read(p) -} - -// DecompressedReader returns a reader that decompresses the body. -// The compression scheme is determined by the Content-Encoding header. -// Closing the returned reader is the caller's responsibility. -func DecompressedReader(request *http.Request) (io.ReadCloser, error) { - var wrapper io.ReadCloser - var err error - - switch request.Header.Get("Content-Encoding") { - case "gzip": - wrapper, err = gzip.NewReader(request.Body) - case "identity", "": - return request.Body, nil - default: - return nil, ErrContentEncodingNotSupported - } - - return &wrapperCloserReader{ - wrapped: request.Body, - wrapper: wrapper, - }, err -} - -// CompressBuffer GZip-compresses the payload into a buffer, and returns it. -func CompressBuffer(payload []byte) *bytes.Buffer { - var bodyBuf bytes.Buffer - compressor := gzip.NewWriter(&bodyBuf) - compressor.Write(payload) - compressor.Close() - return &bodyBuf -} diff --git a/pkg/shaman/httpserver/logging.go b/pkg/shaman/httpserver/logging.go deleted file mode 100644 index a4ec51fa..00000000 --- a/pkg/shaman/httpserver/logging.go +++ /dev/null @@ -1,29 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package httpserver - -import ( - "github.com/sirupsen/logrus" -) - -var packageLogger = logrus.WithField("package", "shaman/httpserver") diff --git a/pkg/shaman/httpserver/testroutes.go b/pkg/shaman/httpserver/testroutes.go deleted file mode 100644 index 0432cce6..00000000 --- a/pkg/shaman/httpserver/testroutes.go +++ /dev/null @@ -1,72 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package httpserver - -import ( - "fmt" - "net/http" - - "git.blender.org/flamenco/pkg/shaman/jwtauth" - "github.com/gorilla/mux" - "github.com/sirupsen/logrus" -) - -var userInfo = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - tokenSubject, ok := jwtauth.SubjectFromContext(r.Context()) - if !ok { - fmt.Fprintf(w, "You are unknown to me") - return - } - - fmt.Fprintf(w, "You are subject %s", tokenSubject) -}) - -// RegisterTestRoutes registers some routes that should only be used for testing. -func RegisterTestRoutes(r *mux.Router, auther jwtauth.Authenticator) { - // On the default page we will simply serve our static index page. - r.Handle("/", http.FileServer(http.Dir("./views/"))) - - // We will setup our server so we can serve static assest like images, css from the /static/{file} route - r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir("./static/")))) - - getTokenHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - tokenString, err := auther.GenerateToken() - if err != nil { - logger := packageLogger.WithFields(logrus.Fields{ - logrus.ErrorKey: err, - "remoteAddr": r.RemoteAddr, - "requestURI": r.RequestURI, - "requestMethod": r.Method, - }) - logger.Warning("unable to sign JWT") - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf("error signing token: %v", err))) - return - } - - w.Write([]byte(tokenString)) - }) - - r.Handle("/get-token", getTokenHandler).Methods("GET") - r.Handle("/my-info", auther.Wrap(userInfo)).Methods("GET") -} diff --git a/pkg/shaman/jwtauth/dummy.go b/pkg/shaman/jwtauth/dummy.go index 74e08052..25f090e3 100644 --- a/pkg/shaman/jwtauth/dummy.go +++ b/pkg/shaman/jwtauth/dummy.go @@ -9,3 +9,7 @@ package jwtauth type Authenticator interface { } + +type AlwaysDeny struct{} + +var _ Authenticator = (*AlwaysDeny)(nil) diff --git a/pkg/shaman/logging.go b/pkg/shaman/logging.go deleted file mode 100644 index 0bcfa268..00000000 --- a/pkg/shaman/logging.go +++ /dev/null @@ -1,29 +0,0 @@ -/* (c) 2019, Blender Foundation - Sybren A. Stüvel - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package shaman - -import ( - "github.com/sirupsen/logrus" -) - -var packageLogger = logrus.WithField("package", "shaman") diff --git a/pkg/shaman/server.go b/pkg/shaman/server.go index 25a0f633..6def6ee9 100644 --- a/pkg/shaman/server.go +++ b/pkg/shaman/server.go @@ -23,15 +23,17 @@ package shaman import ( + "context" + "io" "sync" + "git.blender.org/flamenco/pkg/api" "git.blender.org/flamenco/pkg/shaman/checkout" "git.blender.org/flamenco/pkg/shaman/config" "git.blender.org/flamenco/pkg/shaman/fileserver" "git.blender.org/flamenco/pkg/shaman/filestore" - "git.blender.org/flamenco/pkg/shaman/httpserver" "git.blender.org/flamenco/pkg/shaman/jwtauth" - "github.com/gorilla/mux" + "github.com/rs/zerolog/log" ) // Server represents a Shaman Server. @@ -49,9 +51,8 @@ type Server struct { // NewServer creates a new Shaman server. func NewServer(conf config.Config, auther jwtauth.Authenticator) *Server { - if !conf.Enabled { - packageLogger.Warning("Shaman server is disabled") + log.Info().Msg("Shaman server is disabled") return nil } @@ -60,14 +61,14 @@ func NewServer(conf config.Config, auther jwtauth.Authenticator) *Server { fileServer := fileserver.New(fileStore) shamanServer := &Server{ - conf, - auther, - fileStore, - fileServer, - checkoutMan, + config: conf, + auther: auther, + fileStore: fileStore, + fileServer: fileServer, + checkoutMan: checkoutMan, - make(chan struct{}), - sync.WaitGroup{}, + shutdownChan: make(chan struct{}), + wg: sync.WaitGroup{}, } return shamanServer @@ -76,32 +77,70 @@ func NewServer(conf config.Config, auther jwtauth.Authenticator) *Server { // Go starts goroutines for background operations. // After Go() has been called, use Close() to stop those goroutines. func (s *Server) Go() { - packageLogger.Info("Shaman server starting") + log.Info().Msg("Shaman server starting") s.fileServer.Go() if s.config.GarbageCollect.Period == 0 { - packageLogger.Warning("garbage collection disabled, set garbageCollect.period > 0 in configuration") + log.Warn().Msg("garbage collection disabled, set garbageCollect.period > 0 in configuration") } else if s.config.GarbageCollect.SilentlyDisable { - packageLogger.Debug("not starting garbage collection") + log.Debug().Msg("not starting garbage collection") } else { s.wg.Add(1) go s.periodicCleanup() } } -// AddRoutes adds the Shaman server endpoints to the given router. -func (s *Server) AddRoutes(router *mux.Router) { - s.checkoutMan.AddRoutes(router, s.auther) - s.fileServer.AddRoutes(router, s.auther) - - httpserver.RegisterTestRoutes(router, s.auther) -} - // Close shuts down the Shaman server. func (s *Server) Close() { - packageLogger.Info("shutting down Shaman server") + log.Info().Msg("shutting down Shaman server") + close(s.shutdownChan) + s.fileServer.Close() s.checkoutMan.Close() s.wg.Wait() } + +// Checkout creates a directory, and symlinks the required files into it. The +// files must all have been uploaded to Shaman before calling this. +func (s *Server) Checkout(ctx context.Context, checkoutID string, checkout api.ShamanCheckout) error { + return nil +} + +// Requirements checks a Shaman Requirements file, and returns the subset +// containing the unknown files. +func (s *Server) Requirements(ctx context.Context, requirements api.ShamanRequirements) (api.ShamanRequirements, error) { + return requirements, nil +} + +var fsStatusToApiStatus = map[filestore.FileStatus]api.ShamanFileStatusStatus{ + filestore.StatusDoesNotExist: api.ShamanFileStatusStatusUnknown, + filestore.StatusUploading: api.ShamanFileStatusStatusUploading, + filestore.StatusStored: api.ShamanFileStatusStatusStored, +} + +// Check the status of a file on the Shaman server. +// status (stored, currently being uploaded, unknown). +func (s *Server) FileStoreCheck(ctx context.Context, checksum string, filesize int64) api.ShamanFileStatusStatus { + status := s.fileServer.CheckFile(checksum, filesize) + apiStatus, ok := fsStatusToApiStatus[status] + if !ok { + log.Warn(). + Str("checksum", checksum). + Int64("filesize", filesize). + Int("fileserverStatus", int(status)). + Msg("shaman: unknown status on fileserver") + return api.ShamanFileStatusStatusUnknown + } + return apiStatus +} + +// Store a new file on the Shaman server. Note that the Shaman server can return +// early when another client finishes uploading the exact same file, to prevent +// double uploads. +func (s *Server) FileStore(ctx context.Context, file io.ReadCloser, checksum string, filesize int64, canDefer bool, originalFilename string) error { + err := s.fileServer.ReceiveFile(ctx, file, checksum, filesize, canDefer) + // TODO: Maybe translate this error into something that can be understood by + // the caller without relying on types declared in the `fileserver` package? + return err +}