Manager: more work on porting Shaman code to Flamenco 3
This commit is contained in:
Sybren A. Stüvel
parent
4df0543661
commit
6f35b3303d
@ -89,9 +89,7 @@ type Shaman interface {
|
||||
Requirements(ctx context.Context, requirements api.ShamanRequirements) (api.ShamanRequirements, error)
|
||||
|
||||
// Check the status of a file on the Shaman server.
|
||||
// TODO: instead of an integer, return a constant that indicates the actual
|
||||
// status (stored, currently being uploaded, unknown).
|
||||
FileStoreCheck(ctx context.Context, checksum string, filesize int64) (int, error)
|
||||
FileStoreCheck(ctx context.Context, checksum string, filesize int64) (api.ShamanFileStatusStatus, error)
|
||||
|
||||
// Store a new file on the Shaman server. Note that the Shaman server can
|
||||
// return early when another client finishes uploading the exact same file, to
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"github.com/labstack/echo/v4"
|
||||
|
||||
"git.blender.org/flamenco/pkg/api"
|
||||
"git.blender.org/flamenco/pkg/shaman/fileserver"
|
||||
)
|
||||
|
||||
// Create a directory, and symlink the required files into it. The files must all have been uploaded to Shaman before calling this endpoint.
|
||||
@ -70,11 +71,11 @@ func (f *Flamenco) ShamanFileStoreCheck(e echo.Context, checksum string, filesiz
|
||||
|
||||
// TODO: actually switch over the actual statuses, see the TODO in the Shaman interface.
|
||||
switch status {
|
||||
case 0: // known
|
||||
case api.ShamanFileStatusStatusStored:
|
||||
return e.String(http.StatusOK, "")
|
||||
case 1: // being uploaded
|
||||
case api.ShamanFileStatusStatusUploading:
|
||||
return e.String(420 /* Enhance Your Calm */, "")
|
||||
case 2: // unknown
|
||||
case api.ShamanFileStatusStatusUnknown:
|
||||
return e.String(http.StatusNotFound, "")
|
||||
}
|
||||
|
||||
@ -108,7 +109,21 @@ func (f *Flamenco) ShamanFileStore(e echo.Context, checksum string, filesize int
|
||||
canDefer, origFilename,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Warn().Err(err).Msg("Shaman: checking stored file")
|
||||
if err == fileserver.ErrFileAlreadyExists {
|
||||
return e.String(http.StatusAlreadyReported, "")
|
||||
}
|
||||
|
||||
logger.Warn().Err(err).Msg("shaman: checking stored file")
|
||||
if sizeErr, ok := err.(fileserver.ErrFileSizeMismatch); ok {
|
||||
return sendAPIError(e, http.StatusExpectationFailed,
|
||||
"size mismatch, expected %d bytes, received %d bytes",
|
||||
sizeErr.DeclaredSize, sizeErr.ActualSize)
|
||||
}
|
||||
if checksumErr, ok := err.(fileserver.ErrFileChecksumMismatch); ok {
|
||||
return sendAPIError(e, http.StatusExpectationFailed,
|
||||
"checksum mismatch, expected %d bytes, received %d bytes",
|
||||
checksumErr.DeclaredChecksum, checksumErr.ActualChecksum)
|
||||
}
|
||||
return sendAPIError(e, http.StatusInternalServerError, "unexpected error: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@ -340,11 +340,11 @@ paths:
|
||||
description: Size of the file in bytes.
|
||||
responses:
|
||||
"200":
|
||||
description: The file is known to the server.
|
||||
"420":
|
||||
description: The file is currently being uploaded to the server.
|
||||
"404":
|
||||
description: The file does not exist on the server.
|
||||
description: Normal response.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/ShamanFileStatus'
|
||||
default:
|
||||
description: unexpected error
|
||||
content:
|
||||
@ -402,10 +402,8 @@ paths:
|
||||
`X-Shaman-Can-Defer-Upload: true` was sent in the request.
|
||||
"409":
|
||||
description: Checkout already exists.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Error'
|
||||
"417":
|
||||
description: There was a mismatch between the request parameters and the actual file size or checksum of the uploaded file.
|
||||
default:
|
||||
description: unexpected error
|
||||
content:
|
||||
@ -722,6 +720,15 @@ components:
|
||||
s: 127
|
||||
p: logging.go
|
||||
|
||||
ShamanFileStatus:
|
||||
type: object
|
||||
description: Status of a file in the Shaman storage.
|
||||
properties:
|
||||
"status":
|
||||
type: string
|
||||
enum: [unknown, uploading, stored]
|
||||
required: [status]
|
||||
|
||||
securitySchemes:
|
||||
worker_auth:
|
||||
description: Username is the worker ID, password is the secret given at worker registration.
|
||||
|
||||
@ -1383,6 +1383,7 @@ func (r ShamanCheckoutRequirementsResponse) StatusCode() int {
|
||||
type ShamanFileStoreCheckResponse struct {
|
||||
Body []byte
|
||||
HTTPResponse *http.Response
|
||||
JSON200 *ShamanFileStatus
|
||||
JSONDefault *Error
|
||||
}
|
||||
|
||||
@ -2024,6 +2025,13 @@ func ParseShamanFileStoreCheckResponse(rsp *http.Response) (*ShamanFileStoreChec
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200:
|
||||
var dest ShamanFileStatus
|
||||
if err := json.Unmarshal(bodyBytes, &dest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response.JSON200 = &dest
|
||||
|
||||
case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true:
|
||||
var dest Error
|
||||
if err := json.Unmarshal(bodyBytes, &dest); err != nil {
|
||||
|
||||
@ -18,85 +18,86 @@ import (
|
||||
// Base64 encoded, gzipped, json marshaled Swagger object
|
||||
var swaggerSpec = []string{
|
||||
|
||||
"H4sIAAAAAAAC/+w8227cRpa/UuAskAmWfdHFkq2n1dhxIiOJhUieLBAbcpE83V1WsYqpKqrdMQTMR+yf",
|
||||
"7A6wDztP+wOeP1qcupDFJltqJZLj2d08BC2yLud+pz8kuSwrKUAYnRx9SHS+gJLan8das7mA4pzqS/y7",
|
||||
"AJ0rVhkmRXLUeUuYJpQY/EU1YQb/VpADu4KCZCtiFkB+lOoS1DhJk0rJCpRhYG/JZVlSUdjfzEBpf/yT",
|
||||
"gllylPxh0gI38ZBNnroNyXWamFUFyVFClaIr/PudzHC3f6yNYmLun19UiknFzCpawISBOaiwwj0d2C5o",
|
||||
"Ofzi5jO1oaa+FR2k35lbiRhRfbkZkLpmBb6YSVVSkxy5B+n6wus0UfBzzRQUydFPYRESx+PSwBahsEal",
|
||||
"iCQxVGnLrzfNvTJ7B7lBAI+vKOM04/BCZmdgDILTk5wzJuYciHbviZwRSl7IjOBpekBAFpLl7mf3nB8X",
|
||||
"IMicXYFICWclM1bOrihnBf6/Bk2MxGcaiD9kTF4KviK1RhjJkpkFcUSzl+PdjQj2iL8ubAXMaM1NH67z",
|
||||
"BRD/0sFB9EIuhQeG1BoUWSLsBRhQJRP2/gXTgSRjd3x05vAVzZOJkZIbVvmLmGgvQnlUM5qDPRQKZhB1",
|
||||
"d6KHf0a5hrRPXLMAhUBTzuWS4NZ1QAmdGVyzAPJOZmRBNckABNF1VjJjoBiTH2XNC8LKiq9IARzcNs4J",
|
||||
"vGfaHUj1pSYzqdzR72SWEioKNCCyrBjHNcyMX4tW0DMpOVBhMbqivE+f05VZSEHgfaVAayYt8TMguLqm",
|
||||
"BgqkkVSFQzDwASwmXdY1cDW8SfuicQmrPgwnBQjDZgyUP6QR+ZSUtTYITy3Yz7UTRM+0d14RBu9BxaBq",
|
||||
"PqALx2JF4L1RlFA1r0u0MEHesmo1xo16fCZLOHW6tfrjlyRHNtQaClyZK6AGHKpe/1YRDK2Kt5blDiLE",
|
||||
"yhIKRg3wFVGARxFqUS1gxgTDDSkaAns9XplamsjaeIioMiyvOVUNHzbIg66zYD5vsroDhurM72xU/c4n",
|
||||
"nPvtV0yzdSUzqr6JQKi4XdXy8vDqxBlIJFZQK0X+yNklEEr+xEGgENOiGEnx5ZicgcHj3lqGvHVmxvlj",
|
||||
"KpwtEJQ3d5gFNXh1zQvxhRXIxlKBKKwB0cOEXnMxqAB+0ZZu4azl05p3qLMRvnHi4BQi8Jw8rZUCYfiK",
|
||||
"SLTjNJxrNSyy5HpM3n5zfPbNV88unp98+9XF6fH5N29dlFIwBbmRakUqahbkn8nb18nkD/a/18lbQqsK",
|
||||
"SVo4tEHUJeI3YxwucH2SJgVT4ad97D3qguoFFBftyjcDCrxJaPoG3lMgwj6yGs59UU1OngV9tmij0HiR",
|
||||
"GJPvJRGg0dZpo+rc1Ao0+aN1XzolBcvxKqoY6C8JVUB0XVVSmXXUPfApRjZ7u4g0l9QkqZWFW5Ecxi54",
|
||||
"+/ZOFyUyTb6jgs5BORfAjFV9WqKBHggNOM2A3y1k88TcPtwcCml60cCaOniRcOBFd96mG0itAeP+LdMm",
|
||||
"CIOV7s1069MohHG/DuPzjkXcgG57xRCCIV7voeVfEAXopa3LokS74NBHmdYSvYe8NnBbHrE5SG8EKHod",
|
||||
"wBtmXLRlCKOvlJIKD1vPZAroROdBY/qpQQla0/kQvGsA2TPb9UPQPOe0BJHLP4PSPljckjJX7Y6boQgL",
|
||||
"vV4NQfHCpV6U85ez5OinmyXsLMSHuOs67RHSxiJDEoMvbDTHStCGlhXao0DughoY4Zuh0IkNHPfq1cmz",
|
||||
"4GZe2OzolsRq25wOTUWT0tVVcc/YrHHHQhpo1t7XAPvm+o1j0HdgaEENtYwqCht2UX7aoX0P47U4U2XM",
|
||||
"KKpWpPSHeberx+Q7qaziVhzexz4npwK9Vikx/rcWq0YtJ2/pOBvnb4mQxtEhhMmXYENPeE/xLC/QVtCO",
|
||||
"krNKMQPkuWLzBXohjFHGUFLGEepVpkD8S+ZdoFTzsMLpQHJmF5Az89//dQU8MmwdQT6LfMQwnVw0N7i3",
|
||||
"EZDgQGlu2JXNnKnIkQIuia44GP9bOGIxKUYzytyK5kdFMURP0uTnGmr7g6p8wa6in84/u+NHKBnW7ftD",
|
||||
"Og/sb3dKjSQaxZcnabKkNskbzaQaYSSjBx38DzBn2oCCwhnjvsmhRYGJ16BAcarNhSVKt3ISOW+WX242",
|
||||
"55waVJJh7y5nZknVBte/le46lFr1bVztRVMF6brSWwsFv6lq09AibYgaV28CMdIkd6GxhTJZp3JEmQ0Y",
|
||||
"Ddn0M8hrxcxqg7/b2ond5L3OFrSk4ukC8ktZDxRTMKGRM2KF0RVszAKYImffHO8+OiA5btR1mRLNfrHx",
|
||||
"b7YyoF34WIBGEAiXuTMwPqfK/W1tLrBmbhT8bH1Ynhwle4+y6f6TnXz3MJvu7e0VO7Ns/9Esnx4+fkJ3",
|
||||
"dnM6Pch2ioP9abH76ODJ4eNp9nh6WMCj6X5xON19AlPkEprzJtEdzyUyITna2d/dR+eHtxzsZYe7+cFe",
|
||||
"9mR/d39W7OxlT/YOp7PsYDo9eDJ9PM336M6jw53DfLZHi/393YO9R9nO48P8gD5+8mh6+MTfwuV8jolS",
|
||||
"c8XuoTX+XcZZ9CIJXvO+A0zoEjv4TGTKYJmif8JzxiO6+/Spr6EbNnZ4OxBQrQdNFnmEY0jibgxlkTSb",
|
||||
"xfQHt7IMxfJfJapWNDsYPaz03b+k/W6Cda/ycX+yEUezg7luq/xtaRRTmxC6rwlAGcVpDxf5+Bd7H/+N",
|
||||
"/P0vH//68W8f/+PjX//+l4//+fFvH/897gwcPZp282Z/y0VeFslR8sH/eY1OaFGLywtkR3K0hzgZRXNz",
|
||||
"QeuCyRA1IXOtAThKJsrunOjZ5J3M0AeDgJ3dvbE9Mo6GT7//Gv+sdHK0u58mM8zPdHKU7Ix2UMZZSeeg",
|
||||
"L6S6uGIFSOS9fZKkiaxNVRtXl4H3BoRLeZJxZaMmB8GFW9UFyV3SABXJh2bIqpFHfOS2JD11iPl4S7rQ",
|
||||
"hObbtpuawiIyZ6D3FLHrtkwlLI0Knzf7cx+P+IZQA9WQbkTdrTuExE3w20SrGL60wfFAqOvD5KFwFWF4",
|
||||
"ZZOigTpb847Y2qswmJ9QX2VAHXXplCvfW0TI63o63T0gXM61q83axiczX2hfq/BtgrWQOIp4uzC8FDDi",
|
||||
"TPhKuShYjhcuFxRPzJuK58KWJjFxCr7FXjwmL69ALdE2aFIpuGKy1nzlcAmXNknakFXlcqBJ962cEwQq",
|
||||
"6szgbSlZMs4xnQuFUgTaksJeCFRx5soz/bi4Iwvb9kSHcjTHHZeGKGqGqx6/PomAXIEZfvUbk4E1RfI3",
|
||||
"deL4wSuiPODNRnqcsbl4eVdKhLzgYnMx6N7RjnKaDdj2oLoBa0MNPF1QMYc+6k5jL1pDcafkb51b64dt",
|
||||
"BVSxCap7gOUWCLpGVxuqjCsV0CW9tBml5gAVBh82w0sTvahNIZe24wPar5azGVqCAdvqlMXmiGcItUNv",
|
||||
"aQG4oDX6+F7NTYNC3qO5RRPmFpOTZympqNZLqYrwymmH6/ATasJSFak92hlLL9ucoprlreFZGFMl1wgj",
|
||||
"EzPpCrTC0Ny0NdGmdkrOgaLy1Yr7nfpoMpmF8IzJSb8U9oNrvT2nqiSlq76T49MTzL1ZDkJDdM/Xp99e",
|
||||
"7fXOXy6X47moMVqb+D16Mq/4aG88HYMYL0zpalTM8A60/rokKuEmO+PpeIqrZQWCVgxDO/sIfaNZWM5M",
|
||||
"aMVspGVlUmpLCpRMS8yTwrXfSmZcNdRL+p9ksQrkA2H30KrizCXVk3faWQ0nt7dJdbf0e92jqm0NSR8m",
|
||||
"J7HQY/RotUBXEimFN+1Op/cG2Q0ALakmus5z0LOa8xVxgwl2isC77CtW1JS7WYbx2nTIvUDnajAD8NkX",
|
||||
"JJRYrErWZUnVqmEmoUTA0naP0Jc3UuRbRlGPxbptilGjbero5E3nuBehB+1GKkAUlWTCWHwb0Zo03mEO",
|
||||
"A/L1NZim0fWAzOx31QZI1yxqO2trBPwaDOG97pttTNnUvtucvIF07VUN+d+1I08d+n14J7MLVlxvJOFz",
|
||||
"MPnCaWjc2/rpQ8IQK19c8ZbHHdZTpDSi4211yTe/j9JZq91lh8XcviA0c8MhlndbyK3bJApvO0uEPJA9",
|
||||
"Cn02yeyfmw7Yg5FivY83QBaBnOIkgDAgrEiQRsI8Xs1gzXeN2wjEwgx1jVgufHBdm1r7iSAjXY3G/cU0",
|
||||
"JhY1RVNI2+s0qCsM/QNZnb+eKN8tGC3bZsGg6wltBd9UeBj/M5A6DBC6Tf8C9J/UFfUaLNvIwif0ObWA",
|
||||
"9xXkBgoCfk0sQgF873iWgZ9B6vyDNwObHEtQYtudel2iNJuLkZzNbohiMBWazfrqut+PSD8/QvqQ2pr0",
|
||||
"TjD90xs0xi3NvqPqMo6iqSYhWL+F2k8p973YoO+YxnsDEgKDS2GH0mD1hQIyl25Y1x4/HmaJuIUj4kGV",
|
||||
"2l+xWZ2betyn1OV+lvoPocxby+BxbRYgjCta+dIYSkNoti2beZ17FkgFtFjhKjzPNfw65TrWMrwvrsZX",
|
||||
"Awf9fcSy5PeWDAspye170pYertNNxoxs3vF5i9TdxcOFJMswRbsABW7SdbWBCMNyMMqjQs2g8Roo6jyo",
|
||||
"IYsvGiDv941rdHhuYc/+d/k9b8893xwRxuQcY9Pcfm+Q2elYmqPB4FC4eN8V670taZsHHVlJiVRouQJV",
|
||||
"gn0BNeIyp9yaNsr1fduzK+hgU+ueqBr/FdYG95ovoKg5nLvpj4fLq+NvwgYYa78GiwsKmwzV99J/+NGd",
|
||||
"4bb5RRjxvE6T/ene/ZWeOuMsA8Cfggq1jWcgmDOa+9MnA58eOQFkmghpgqdzXS0nTinRMry2389AZ5bV",
|
||||
"oW47uUTIpUN1d+/TupagRVQglDIzlAkbdlvoUpLVxo2cz6X9DEhIa2edtt1RY1+602lzfkSN21TJypT2",
|
||||
"Aq4Gyk6Rhkw+2D6CL58M60rUD9ymguIP/O0llPt3FxEmm3TRx0NMOBBDDePO3uJ8AeGspTWtOVTBow6q",
|
||||
"yLnvT1qP7K1GLEaOaVZPTPdsqzPx+f8obulV2yp2vVKzqlhuyyRxZ7dScq5A69QPy/qvnxSZUcZrBbf6",
|
||||
"luBRNIiiUw1DcofT0YphROTURNv5pUmYwJq4ceHJh/Dg5NkNCrM2o7eN0rTn3qg3n0hP1hAY4HxndqsJ",
|
||||
"LWVtxr9OXcJdVqD9bHbcQoj15mFluoGEcpcx2Q8v9fhzKhg9dR8fRh9nuXxOr0rOhCtxBCZ4HrnOi8Go",
|
||||
"D/wj66so52RBr8B9iFpXXFI/pOCEgGQws3PqlPPmc9bWpbSa5zRmTfPOPECU6FhgLDCdyVKqgA5rnlqb",
|
||||
"IdxG5Tpzhw+pIp2LtlWTT1pO2RLOOvMMQq4giaEgMeXTYI6dDACpBQbjwqH2eSmHHY4kNAhwjLsFN3wu",
|
||||
"XUllNFkuWL7wHKKqQexW0T7GKBWvyTnDZCnKr7sHtgG7FL5Pr67CF3tusRR+rTaM8xaESB/sed7/6Lq8",
|
||||
"nnywT9gvcO20AYmjNynGc8bhzEgFT70ArnmkradM7Sf/G9yXrss7Oa+0/w8d/ALxZZ2B34FbAwW2ubWd",
|
||||
"bd3QnutHYw4ETZyge4sVNW32N8V9dmMhwYVm1n90ee/zl1vubUfZMkC1i43z2lmfle7Fs3+2hGAR8gTw",
|
||||
"Sulhd99K99XsJiPfyPL/bTFOh0TH26IQOmv3xToztjtZwAyUl6LGlVtq2KDgdbI7ffw6aUs5dnLRprrC",
|
||||
"yiBRYGqFaYn9TN5EohpiJTfu0VGYDsNdkky5lu4MLUuQAghwvSbyQ2BaabEEXAAtbIvMk/BfR+6a0VMq",
|
||||
"Rs8Qz9Ere0AyQMPoo/whGkrF5kxQbu/E88fkZObHQ7mMx0l9XJEigcOYJxM5rwv3D2a0SmonPlNv7ZEX",
|
||||
"lNkVBWS1+wJlC9xeesBGzz1gyb2kBjI3YEbaKKB2vrMZrE9eYIBowzS306mzWHkVGsigvUh8oZstD5kP",
|
||||
"7E4f32BBF7SVy05wOybfS5t2Uv+vZViGoExm4Pjs5dvLXVcwPV8rJXPQliJd6+z8+tuNEnlEkAhv3YyU",
|
||||
"U9ZYmFAQ/j/ViTIItPS+DrbZj3Q4uvbSsncmVc4yviI5l9qVHL45Pz9FMRVgv+Z0TA/VFm9IZ0wwvQDd",
|
||||
"MUdA4D3NDdG0BB9QGmlHyXFLIWuM9dwGvdHDxVUQXNnmCH3sfFkFfzs/58YiJ0nUBur9azrdoZ/eEBsz",
|
||||
"Gvhs3JoZO9rSt4gvZBa6lLZc8nMNioFOo8G2dG1OaNyZptIDhx6fnnRH6+ImlSzLWviZfrS066BHx/tq",
|
||||
"z4ALdvQ7Pj1J7UVWcloeeoSsXcG/38msSUV1dL7n1/Wb6/8JAAD//+OqSK73TQAA",
|
||||
"H4sIAAAAAAAC/+w86W4cR3qvUugN4DXSc/AQKfFXuJJlU7AtwqTWASyCqu7+ZqbE6qp2VTVHY4HAPkTe",
|
||||
"JFkgP7K/8gLaNwq+OvqYriFHNilrk/iHMeyu47vv1vskl2UlBQijk6P3ic4XUFL781hrNhdQnFN9hX8X",
|
||||
"oHPFKsOkSI56bwnThBKDv6gmzODfCnJg11CQbEXMAsiPUl2BGidpUilZgTIM7C25LEsqCvubGSjtj39S",
|
||||
"MEuOkj9MWuAmHrLJU7chuUkTs6ogOUqoUnSFf7+VGe72j7VRTMz988tKMamYWXUWMGFgDiqscE8j2wUt",
|
||||
"4y9uP1Mbauo70UH6nbmViBHVV5sBqWtW4IuZVCU1yZF7kK4vvEkTBT/XTEGRHP0UFiFxPC4NbB0U1qjU",
|
||||
"IUkXqrTl10Vzr8zeQm4QwONryjjNOLyQ2RkYg+AMJOeMiTkHot17ImeEkhcyI3iajgjIQrLc/eyf8+MC",
|
||||
"BJmzaxAp4axkxsrZNeWswP/XoImR+EwD8YeMyUvBV6TWCCNZMrMgjmj2cry7EcEB8deFrYAZrbkZwnW+",
|
||||
"AOJfOjiIXsil8MCQWoMiS4S9AAOqZMLev2A6kGTsju+cGb+ieTIxUnLDKn8RE+1FKI9qRnOwh0LBDKLu",
|
||||
"TvTwzyjXkA6JaxagEGjKuVwS3LoOKKEzg2sWQN7KjCyoJhmAILrOSmYMFGPyo6x5QVhZ8RUpgIPbxjmB",
|
||||
"d0y7A6m+0mQmlTv6rcxSQkWBBkSWFeO4hpnxa9EKeiYlByosRteUD+lzujILKQi8qxRozaQlfgYEV9fU",
|
||||
"QIE0kqpwCAY+gMWkz7oGroY36VA0rmA1hOGkAGHYjIHyhzQin5Ky1gbhqQX7uXaC6Jn21itC9B5UDKrm",
|
||||
"EV04FisC74yihKp5XaKFCfKWVasxbtTjM1nCqdOt1R+/JDmyodZQ4MpcATXgUPX6t+rA0Kp4a1k+QoRY",
|
||||
"WULBqAG+IgrwKEItqgXMmGC4IUVDYK/HK1NLE1kbDxFVhuU1p6rhwwZ50HUWzOdtVjdiqM78zkbVP/qE",
|
||||
"c7/9mmm2rmRG1bcRCBW3r1peHl6dOAOJxApqpcgfObsCQsmfOAgUYloUIym+HJMzMHjcG8uQN87MOH9M",
|
||||
"hbMFgvLmDrOgBq+ueSG+sALZWCoQhTUgOk7oNReDCuAXbekWzlo+rXmHOhvhGycOTiECz8nTWikQhq+I",
|
||||
"RDtOw7lWwzqWXI/Jm2+Oz7756tnl85Nvv7o8PT7/5o2LUgqmIDdSrUhFzYL8M3nzOpn8wf73OnlDaFUh",
|
||||
"SQuHNoi6RPxmjMMlrk/SpGAq/LSPvUddUL2A4rJdeRFR4E1CMzTwngId7DtWw7kvqsnJs6DPFm0UGi8S",
|
||||
"Y/K9JAI02jptVJ2bWoEmf7TuS6ekYDleRRUD/SWhCoiuq0oqs466Bz7FyGZvF5HmkpoktbJwJ5Jx7IK3",
|
||||
"b+90USLT5Dsq6ByUcwHMWNWnJRroSGjAaQb840I2T8ztw81YSDOIBtbUwYuEA69z5126gdSKGPdvmTZB",
|
||||
"GKx0b6bbkEYhjPt1GJ/3LOIGdNsrYgiGeH2Aln9BFKCXti6LEu2CQx9lWkv0DvLawF15xOYgvRGgzusA",
|
||||
"XpxxnS0xjL5SSio8bD2TKaAXnQeNGaYGJWhN5zF41wCyZ7brY9A857QEkcs/g9I+WNySMtftjtuhCAu9",
|
||||
"XsWgeOFSL8r5y1ly9NPtEnYW4kPcdZMOCGljkZjE4AsbzbEStKFlhfYokLugBkb4JhY6schxr16dPAtu",
|
||||
"5oXNju5IrLbN6dBUNCldXRX3jM0adyykgWbtfQ2wFzcXjkHfgaEFNdQyqihs2EX5aY/2A4zX4kyVMaOo",
|
||||
"WpHSH+bdrh6T76SyiltxeNf1OTkV6LVKifG/tVg1ajl5Q8fZOH9DhDSODiFMvgIbesI7imd5gbaCdpSc",
|
||||
"VYoZIM8Vmy/QC2GMMoaSMo5QrzIF4l8y7wKlmocVTgeSM7uAnJn//q9r4B3D1hPks46PiNPJRXPRvY2A",
|
||||
"BAdKc8OubeZMRY4UcEl0xcH438IRi0kxmlHmVjQ/KoohepImP9dQ2x9U5Qt23fnp/LM7foSSYd2+P6T3",
|
||||
"wP52p9RIolH38iRNltQmeaOZVCOMZHTUwf8Ac6YNKCicMR6aHFoUmHhFBYpTbS4tUfqVk47zZvnVZnPO",
|
||||
"qUEliXt3OTNLqja4/q1016HUqm/jai+bKkjfld5ZKPhNVZuGFmlD1G71JhAjTXIXGlsok3UqdyizAaOY",
|
||||
"TT+DvFbMrDb4u62d2G3e62xBSyqeLiC/knWkmIIJjZwRK4yuYGMWwBQ5++Z499EByXGjrsuUaPaLjX+z",
|
||||
"lQHtwscCNIJAuMydgfE5Ve5va3OBNXOj4Gfrw/LkKNl7lE33n+zku4fZdG9vr9iZZfuPZvn08PETurOb",
|
||||
"0+lBtlMc7E+L3UcHTw4fT7PH08MCHk33i8Pp7hOYIpfQnDeJ7ngukQnJ0c7+7j46P7zlYC873M0P9rIn",
|
||||
"+7v7s2JnL3uydzidZQfT6cGT6eNpvkd3Hh3uHOazPVrs7+8e7D3Kdh4f5gf08ZNH08Mn/hYu53NMlJor",
|
||||
"dg+t8e8zzqLXkeA17xthQp/YwWciU6JliuEJzxnv0N2nT0MN3bCxx9tIQLUeNFnkEY6YxN0ayiJpNosp",
|
||||
"AtMa+DUa2ecuWUfCBGFzO4k2UtE5DONWPXAYtbgScimsK+eSFs684wFQROzxGgbB6W9E4ge3uAwV/1+l",
|
||||
"b1a/emx5WBW6f3X53bTjXoX8/gS8G5JHE/bWgrX1XczPQv6xJgBlJ9h8uPDNv9j78G/k73/58NcPf/vw",
|
||||
"Hx/++ve/fPjPD3/78O/d9sbRo2k/+fe3XOZlkRwl7/2fN+hJF7W4ukR2JEd7iJNRNDeXtC6YDKEfMtda",
|
||||
"saNkouzOiZ5N3soMAwkQsLO7N7ZHdkP60++/xj8rnRzt7qfJDJNMnRwlO6MdlHFW0jnoS6kur1kBEnlv",
|
||||
"nyRpImtT1cYVl+CdAeHytmRcWdvgILh0q/oguUsaoDryoRmyauQRH7ktyUAduny8I+dp8otte2ZNdRSZ",
|
||||
"E2mgddh1V7oVlnaqt7fbSR9U+a5WA1VMNzotuo+I65sIvgm5MQZrI/xIvO5j/VjMjTC8spldpFjYvCO2",
|
||||
"gCwMJlnUl0pQR11O6HoQzk+9rqfT3QPC5dz7LNu9ZeYL7QsuvtexFtd3wvY+DC8FjDgTvtwvCpbjhcsF",
|
||||
"xRPzpmy7sPVVzP6Cb7EXj8nLa1BLtA2aVAqumaw1XzlcwqVNphmzqlxGOo3fyjlBoDrtJbwtJUvGOeak",
|
||||
"odqLQFtS2AuBKs5cjWkY3PdkYdvGbizRdNxxuZSiJl66+fWZEOQKTPzVb8xo1gMOd1MvGYle0UlmLjbS",
|
||||
"44zNxcuPpURIbi43V7TuHe1OYrYB2wFUt2BtqIGnCyrmMETdaexlayg+KoONhoedw7YCqtgE1T3AcgcE",
|
||||
"faOrDVXGBcR0Sa9sWqw5QIXBh01T00QvalO4ANqA9qvlbIaWIB4/a5/oniHUDr2lBeCS1ujjB4VDDQp5",
|
||||
"j+YWTZhbTE6epaSiWi+lKsIrpx1uTIFQE5aqjtqjnbH0sh02qlneGp6FMVVygzAyMZOuyiwMzU1b2G0K",
|
||||
"wOQcKCpfrbjfqY8mk1kIz5icDOt5P7j+4XOqSlK6FgI5Pj1J0oSzHISGzj1fn357vTc4f7lcjueixmht",
|
||||
"4vfoybzio73xdAxivDClK7Qxw3vQ+uuSTh062RlPx1NcLSsQtGIY2tlH6BvNwnJmQitmIy0rk1JbUqBk",
|
||||
"WmKeFK6HWDLjSrpe0v8ki1UgHwi7h1YVZ64yMHmrndVwcnuXVPfr1zcDqtr+lvRhctIVeowerRboSiKl",
|
||||
"8Kbd6fTeILsFoCXVRNd5DnpWc74ibrrCjkJ4l33NippyN5AxXhtxuRfoXCEpAp99QUKdyKpkXZZUrRpm",
|
||||
"EkoELG0LDH15I0W+79VpFFm3TTFqtJ0pnVz0jnsRGuluLgREUUkmjMW3Ea1J4x3mEJGvr8E03boHZOaw",
|
||||
"NRghXbOobQ+uEfBrMIQPWoi2u2ZT+36H9RbStVc15H/bzm316Pf+rcwuWXGzkYTPweQLp6HdBt1P7xOG",
|
||||
"WPkKkbc87rCBIqUdOt5VXL34fZTOWu0+Oyzm9gWhmZtwsbzbQm7dJlF421ki5IHsndBnk8z+uWnjPRgp",
|
||||
"1puREbII5BQnAYSIsCJBGgnzeDXTQd81biMQCzPUNWK58MG1nmrtx5qMdDUa9xfTmFjUFE0hba/ToK4x",
|
||||
"9A9kdf56onzLY7RsOx5R1xN6I74z8jD+J5I6RAjdpn8B+k/qigZdom1k4RP6nFrAuwpyAwUBv6YrQgF8",
|
||||
"73iWgZ9B6vyDi8gmxxKU2HanXpcozeZiJGezW6IYTIVms6G67g8j0s+PkD6ktia9F0z/dIHGuKXZd1Rd",
|
||||
"daNoqkkI1u+g9lPKfUM56Dum8d6AhMDgStjJOlh9oYDMpZs4tseP4ywRd3BEPKhS+ys2q3NTj/uUujzM",
|
||||
"Uv8hlHlrGTyuzQKEcUUrXxpDaQhNnGUzdHTPAqmAFitchee5rmWvXMdahg/F1fhqYNTfd1iW/N6SYSEl",
|
||||
"uX1P2tLDTbrJmJHNOz5vkfp48XAhyTKMAi9AgRvXXW0gQlwORnmnUBM1XpGizoMasu5FEfJ+37hGh+cW",
|
||||
"9ux/l9/z9tzzzRFhTM4xNs3tRxOZHfGlORoMDoWL912x3tuStnnQk5WUSIWWK1Al2BdQIy5zyq1po1zf",
|
||||
"tz27hh42tR6IqvGfkm1wr/kCiprDuRthebi8uvthW4Sx9pO2bkFhk6H6XvqvV/qD6Da/CHOqN2myP927",
|
||||
"v9JTbyYnAvwpqFDbeAaCOaO5P30S+X7KCSDTREgTPJ3rajlxSomW4bX9CAh6A7kOddvJJUIuHaq7e5/W",
|
||||
"tQQtogKhlJmhTNiw20KXkqw2bm5+Lu23TEJaO+u07SM19qU7nTbnd6hxlypZmdJewFWk7NTRkMl720fw",
|
||||
"5ZO4rnT6gdtUUPyBv72Ecv/uooPJJl308RATDsRQw/hob3G+gHDW0prWHKrgUaMqcu77k9Yje6vRFSPH",
|
||||
"NKsnpn+21Znu+f8obulV2yp2vVKzqlhuyyTdzm6l5FyB1qmf+PWfcCkyo4zXCu70LcGjaBBFrxqG5A6n",
|
||||
"oxXDiMipibbzS5MwRjZxM8+T9+HBybNbFGZt0HAbpWnPvVVvPpGerCEQ4XxvdqsJLWVtxr9OXcJdVqD9",
|
||||
"gHm3hdDVm4eV6QYSyl3GZL8e1ePPqWD01H1B2fnCzOVzelVyJlyJIzDB88h1XgxGfeAfWV9FOScLeg3u",
|
||||
"a1o3BegMj58mzGBmh+0p5803ua1LaTXPacya5p15gCjRXYGxwPTGY6kCGtc8tTZDuI3K9eYOH1JFehdt",
|
||||
"qyaftJyyJZx15hmEXEESQ0G6lE+DOXYyAMRPjjrUPi/lsMORhAYB7uJuwQ3ffFdSGU2WC5YvPIeoahC7",
|
||||
"U7SPMUrFa3LOMFnq5Nf9A9uAXQrfp1fX4bNDt1gKv1YbxnkLQkcf7Hne/+i6vJm8t0/YL3DjtAGJozcp",
|
||||
"hpsmlgqeegFc80hbT5naf7dgg/vSdflRzisd/msNv0D3st7Ab+TWQIFtbm1nWy8eXNs6s9uxakS/jvD5",
|
||||
"6U53dq+dMZf9GXPXu3IfbA/V5DYj3cji/20xTGMJgLclIfTV7rN5Zmx3sYAZKNIM6jtXbKlhnfrrZHf6",
|
||||
"+HXSlmLs5KFNVQVfkQxDAlMrTCvst/oterqJddy4hrPs3kX3GO6SXMq1dGdoWYIUQIBre047fRkD00qL",
|
||||
"JeACaGFbXJ6E/zpy14yeUjF6hniOXtkDkggNO/8yQIyGUrE5E5TbO/H8MTmZ+fFOLrvjoD4uSJHAYUyT",
|
||||
"iZzXhftXO1pjbSc2U2+tkReU2RUFZLX7DGYL3F56wEbPPWDJvYT2MjdgRtoooHY+sxmMT15ggGfDLLfT",
|
||||
"qbNYeRWKZMBeJL7QzZaHjOd3p4/jebAVywVt5bIXnI7J99KmjdT/kx2WISiTGTg+e/n2ctcXTM/XSskc",
|
||||
"tKVIBiim4XTnl99slMgjgkR442acnLJ2hQkF4f9TlU4GgJbe17E2+5EeR9deWvbOpMpZxlck51K7ksE3",
|
||||
"5+enKKYC7CeljumhWuIN6YwJphege+YICLyjuSGaluADQiPtKDhuKWSNsZrboDd6uG4VA1e2Mf4QO18W",
|
||||
"wd/Oz7mxxknSaeMM/kmf/tDOYAiNGQ18Nm7NjB1NGVrEFzILXUZb7vi5BsVAp53BtHRtzmfcm4bSkUOP",
|
||||
"T0/6o3HdJpMsy1r4mXy0tOugd4731ZqIC3b0Oz49Se1FVnJaHnqErF3Bv9/KrEklded8z6+bi5v/CQAA",
|
||||
"//+lKaE0fE4AAA==",
|
||||
}
|
||||
|
||||
// GetSwagger returns the content of the embedded swagger specification file
|
||||
|
||||
@ -66,6 +66,15 @@ const (
|
||||
JobStatusWaitingForFiles JobStatus = "waiting-for-files"
|
||||
)
|
||||
|
||||
// Defines values for ShamanFileStatusStatus.
|
||||
const (
|
||||
ShamanFileStatusStatusStored ShamanFileStatusStatus = "stored"
|
||||
|
||||
ShamanFileStatusStatusUnknown ShamanFileStatusStatus = "unknown"
|
||||
|
||||
ShamanFileStatusStatusUploading ShamanFileStatusStatus = "uploading"
|
||||
)
|
||||
|
||||
// Defines values for TaskStatus.
|
||||
const (
|
||||
TaskStatusActive TaskStatus = "active"
|
||||
@ -247,6 +256,14 @@ type ShamanCheckout struct {
|
||||
} `json:"req"`
|
||||
}
|
||||
|
||||
// Status of a file in the Shaman storage.
|
||||
type ShamanFileStatus struct {
|
||||
Status ShamanFileStatusStatus `json:"status"`
|
||||
}
|
||||
|
||||
// ShamanFileStatusStatus defines model for ShamanFileStatus.Status.
|
||||
type ShamanFileStatusStatus string
|
||||
|
||||
// Set of files with their SHA256 checksum and size in bytes.
|
||||
type ShamanRequirements struct {
|
||||
Req []struct {
|
||||
|
||||
@ -46,7 +46,7 @@ type GCStats struct {
|
||||
}
|
||||
|
||||
func (s *Server) periodicCleanup() {
|
||||
defer packageLogger.Debug("shutting down period cleanup")
|
||||
defer log.Debug().Msg("shaman: shutting down period cleanup")
|
||||
defer s.wg.Done()
|
||||
|
||||
for {
|
||||
|
||||
@ -23,24 +23,10 @@
|
||||
package fileserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"git.blender.org/flamenco/pkg/shaman/filestore"
|
||||
)
|
||||
|
||||
var responseForStatus = map[filestore.FileStatus]int{
|
||||
filestore.StatusUploading: 420, // Enhance Your Calm
|
||||
filestore.StatusStored: http.StatusOK,
|
||||
filestore.StatusDoesNotExist: http.StatusNotFound,
|
||||
}
|
||||
|
||||
func (fs *FileServer) checkFile(ctx context.Context, w http.ResponseWriter, checksum string, filesize int64) {
|
||||
func (fs *FileServer) CheckFile(checksum string, filesize int64) filestore.FileStatus {
|
||||
_, status := fs.fileStore.ResolveFile(checksum, filesize, filestore.ResolveEverything)
|
||||
code, ok := responseForStatus[status]
|
||||
if !ok {
|
||||
packageLogger.WithField("fileStoreStatus", status).Error("no HTTP status code implemented")
|
||||
code = http.StatusInternalServerError
|
||||
}
|
||||
w.WriteHeader(code)
|
||||
return status
|
||||
}
|
||||
|
||||
@ -1,29 +0,0 @@
|
||||
/* (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package fileserver
|
||||
|
||||
import (
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var packageLogger = logrus.WithField("package", "shaman/receiver")
|
||||
@ -24,153 +24,156 @@ package fileserver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"git.blender.org/flamenco/pkg/shaman/filestore"
|
||||
"git.blender.org/flamenco/pkg/shaman/hasher"
|
||||
"git.blender.org/flamenco/pkg/shaman/httpserver"
|
||||
"git.blender.org/flamenco/pkg/shaman/jwtauth"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
// receiveFile streams a file from a HTTP request to disk.
|
||||
func (fs *FileServer) receiveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, checksum string, filesize int64) {
|
||||
logger := packageLogger.WithFields(jwtauth.RequestLogFields(r))
|
||||
// ErrFileAlreadyExists indicates that a file already exists in the Shaman
|
||||
// storage. It can also be returned during upload, when someone else succesfully
|
||||
// uploaded the same file at the same time.
|
||||
var ErrFileAlreadyExists = errors.New("uploaded file already exists")
|
||||
|
||||
bodyReader, err := httpserver.DecompressedReader(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
type ErrFileSizeMismatch struct {
|
||||
DeclaredSize int64
|
||||
ActualSize int64
|
||||
}
|
||||
|
||||
func (e ErrFileSizeMismatch) Error() string {
|
||||
return fmt.Sprintf("file size mismatched, declared %d but received %d bytes", e.DeclaredSize, e.ActualSize)
|
||||
}
|
||||
|
||||
type ErrFileChecksumMismatch struct {
|
||||
DeclaredChecksum string
|
||||
ActualChecksum string
|
||||
}
|
||||
|
||||
func (e ErrFileChecksumMismatch) Error() string {
|
||||
return fmt.Sprintf("file SHA256 mismatched, declared %s but received %s", e.DeclaredChecksum, e.ActualChecksum)
|
||||
}
|
||||
|
||||
// ReceiveFile streams a file from a HTTP request to disk.
|
||||
func (fs *FileServer) ReceiveFile(
|
||||
ctx context.Context, bodyReader io.ReadCloser,
|
||||
checksum string, filesize int64, canDefer bool,
|
||||
) error {
|
||||
logger := *zerolog.Ctx(ctx)
|
||||
defer bodyReader.Close()
|
||||
|
||||
originalFilename := r.Header.Get("X-Shaman-Original-Filename")
|
||||
if originalFilename == "" {
|
||||
originalFilename = "-not specified-"
|
||||
}
|
||||
logger = logger.WithField("originalFilename", originalFilename)
|
||||
|
||||
localPath, status := fs.fileStore.ResolveFile(checksum, filesize, filestore.ResolveEverything)
|
||||
logger = logger.WithField("path", localPath)
|
||||
if status == filestore.StatusStored {
|
||||
logger.Info("uploaded file already exists")
|
||||
w.Header().Set("Location", r.RequestURI)
|
||||
http.Error(w, "File already stored", http.StatusAlreadyReported)
|
||||
return
|
||||
logger = logger.With().Str("path", localPath).Logger()
|
||||
|
||||
switch status {
|
||||
case filestore.StatusStored:
|
||||
logger.Info().Msg("uploaded file already exists")
|
||||
return ErrFileAlreadyExists
|
||||
case filestore.StatusUploading:
|
||||
if canDefer {
|
||||
logger.Info().Msg("someone is uploading this file and client can defer")
|
||||
return ErrFileAlreadyExists
|
||||
}
|
||||
}
|
||||
|
||||
if status == filestore.StatusUploading && r.Header.Get("X-Shaman-Can-Defer-Upload") == "true" {
|
||||
logger.Info("someone is uploading this file and client can defer")
|
||||
http.Error(w, "File being uploaded, please defer", http.StatusAlreadyReported)
|
||||
return
|
||||
}
|
||||
logger.Info("receiving file")
|
||||
logger.Info().Msg("receiving file")
|
||||
|
||||
streamTo, err := fs.fileStore.OpenForUpload(checksum, filesize)
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("unable to open file for writing uploaded data")
|
||||
http.Error(w, "Unable to open file", http.StatusInternalServerError)
|
||||
return
|
||||
return fmt.Errorf("opening file for writing uploaded data: %w", err)
|
||||
}
|
||||
|
||||
// clean up temporary file if it still exists at function exit.
|
||||
// Clean up temporary file if it still exists at function exit.
|
||||
defer func() {
|
||||
streamTo.Close()
|
||||
fs.fileStore.RemoveUploadedFile(streamTo.Name())
|
||||
}()
|
||||
|
||||
// Abort this uploadwhen the file has been finished by someone else.
|
||||
// Abort this upload when the file has been finished by someone else.
|
||||
uploadDone := make(chan struct{})
|
||||
uploadAlreadyCompleted := false
|
||||
defer close(uploadDone)
|
||||
receiverChannel := fs.receiveListenerFor(checksum, filesize)
|
||||
go func() {
|
||||
select {
|
||||
case <-receiverChannel:
|
||||
case <-uploadDone:
|
||||
close(receiverChannel)
|
||||
return
|
||||
case <-receiverChannel:
|
||||
}
|
||||
|
||||
logger := logger.WithField("path", localPath)
|
||||
logger.Info("file was completed during someone else's upload")
|
||||
logger.Info().Msg("file was completed during someone else's upload")
|
||||
|
||||
uploadAlreadyCompleted = true
|
||||
err := r.Body.Close()
|
||||
err := bodyReader.Close()
|
||||
if err != nil {
|
||||
logger.WithError(err).Warning("error closing connection")
|
||||
logger.Warn().Err(err).Msg("error closing connection")
|
||||
}
|
||||
}()
|
||||
|
||||
// TODO: pass context to hasher.Copy()
|
||||
written, actualChecksum, err := hasher.Copy(streamTo, bodyReader)
|
||||
if err != nil {
|
||||
if closeErr := streamTo.Close(); closeErr != nil {
|
||||
logger.WithFields(logrus.Fields{
|
||||
logrus.ErrorKey: err,
|
||||
"closeError": closeErr,
|
||||
}).Error("error closing local file after other I/O error occured")
|
||||
logger.Error().
|
||||
AnErr("copyError", err).
|
||||
AnErr("closeError", closeErr).
|
||||
Msg("error closing local file after other I/O error occured")
|
||||
}
|
||||
|
||||
logger = logger.WithError(err)
|
||||
if uploadAlreadyCompleted {
|
||||
logger.Debug("aborted upload")
|
||||
w.Header().Set("Location", r.RequestURI)
|
||||
http.Error(w, "File already stored", http.StatusAlreadyReported)
|
||||
} else if err == io.ErrUnexpectedEOF {
|
||||
logger.Info("unexpected EOF, client probably just disconnected")
|
||||
} else {
|
||||
logger.Warning("unable to copy request body to file")
|
||||
http.Error(w, "I/O error", http.StatusInternalServerError)
|
||||
logger = logger.With().Err(err).Logger()
|
||||
switch {
|
||||
case uploadAlreadyCompleted:
|
||||
logger.Debug().Msg("aborted upload")
|
||||
return ErrFileAlreadyExists
|
||||
case err == io.ErrUnexpectedEOF:
|
||||
logger.Debug().Msg("unexpected EOF, client probably just disconnected")
|
||||
return err
|
||||
default:
|
||||
return fmt.Errorf("unable to copy request body to file: %w", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err := streamTo.Close(); err != nil {
|
||||
logger.WithError(err).Warning("error closing local file")
|
||||
http.Error(w, "I/O error", http.StatusInternalServerError)
|
||||
return
|
||||
return fmt.Errorf("closing local file: %w", err)
|
||||
}
|
||||
|
||||
if written != filesize {
|
||||
logger.WithFields(logrus.Fields{
|
||||
"declaredSize": filesize,
|
||||
"actualSize": written,
|
||||
}).Warning("mismatch between expected and actual size")
|
||||
http.Error(w,
|
||||
fmt.Sprintf("Received %d bytes but you promised %d", written, filesize),
|
||||
http.StatusExpectationFailed)
|
||||
return
|
||||
logger.Warn().
|
||||
Int64("declaredSize", filesize).
|
||||
Int64("actualSize", written).
|
||||
Msg("mismatch between expected and actual size")
|
||||
return ErrFileSizeMismatch{
|
||||
DeclaredSize: filesize,
|
||||
ActualSize: written,
|
||||
}
|
||||
}
|
||||
|
||||
if actualChecksum != checksum {
|
||||
logger.WithFields(logrus.Fields{
|
||||
"declaredChecksum": checksum,
|
||||
"actualChecksum": actualChecksum,
|
||||
}).Warning("mismatch between expected and actual checksum")
|
||||
http.Error(w,
|
||||
"Declared and actual checksums differ",
|
||||
http.StatusExpectationFailed)
|
||||
return
|
||||
logger.Warn().
|
||||
Str("declaredChecksum", checksum).
|
||||
Str("actualChecksum", actualChecksum).
|
||||
Msg("mismatch between expected and actual checksum")
|
||||
return ErrFileChecksumMismatch{
|
||||
DeclaredChecksum: checksum,
|
||||
ActualChecksum: actualChecksum,
|
||||
}
|
||||
}
|
||||
|
||||
logger.WithFields(logrus.Fields{
|
||||
"receivedBytes": written,
|
||||
"checksum": actualChecksum,
|
||||
"tempFile": streamTo.Name(),
|
||||
}).Debug("File received")
|
||||
logger.Debug().
|
||||
Int64("receivedBytes", written).
|
||||
Str("checksum", actualChecksum).
|
||||
Str("tempFile", streamTo.Name()).
|
||||
Msg("File received")
|
||||
|
||||
if err := fs.fileStore.MoveToStored(checksum, filesize, streamTo.Name()); err != nil {
|
||||
logger.WithFields(logrus.Fields{
|
||||
"tempFile": streamTo.Name(),
|
||||
logrus.ErrorKey: err,
|
||||
}).Error("unable to move file from 'upload' to 'stored' storage")
|
||||
http.Error(w,
|
||||
"unable to move file from 'upload' to 'stored' storage",
|
||||
http.StatusInternalServerError)
|
||||
return
|
||||
logger.Error().
|
||||
Err(err).
|
||||
Str("tempFile", streamTo.Name()).
|
||||
Msg("unable to move file from 'upload' to 'stored' storage")
|
||||
return err
|
||||
}
|
||||
|
||||
http.Error(w, "", http.StatusNoContent)
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -1,97 +0,0 @@
|
||||
/* (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package fileserver
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"git.blender.org/flamenco/pkg/shaman/jwtauth"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// AddRoutes adds this package's routes to the Router.
|
||||
func (fs *FileServer) AddRoutes(router *mux.Router, auther jwtauth.Authenticator) {
|
||||
router.Handle("/files/{checksum}/{filesize}", auther.Wrap(fs)).Methods("GET", "POST", "OPTIONS")
|
||||
}
|
||||
|
||||
func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
logger := packageLogger.WithFields(jwtauth.RequestLogFields(r))
|
||||
|
||||
checksum, filesize, err := parseRequestVars(w, r)
|
||||
if err != nil {
|
||||
logger.WithError(err).Warning("invalid request")
|
||||
return
|
||||
}
|
||||
|
||||
logger = logger.WithFields(logrus.Fields{
|
||||
"checksum": checksum,
|
||||
"filesize": filesize,
|
||||
})
|
||||
|
||||
switch r.Method {
|
||||
case http.MethodOptions:
|
||||
logger.Info("checking file")
|
||||
fs.checkFile(r.Context(), w, checksum, filesize)
|
||||
case http.MethodGet:
|
||||
// TODO: make optional or just delete:
|
||||
logger.Info("serving file")
|
||||
fs.serveFile(r.Context(), w, checksum, filesize)
|
||||
case http.MethodPost:
|
||||
fs.receiveFile(r.Context(), w, r, checksum, filesize)
|
||||
default:
|
||||
// This should never be reached due to the router options, but just in case.
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
func parseRequestVars(w http.ResponseWriter, r *http.Request) (string, int64, error) {
|
||||
vars := mux.Vars(r)
|
||||
checksum, ok := vars["checksum"]
|
||||
if !ok {
|
||||
http.Error(w, "missing checksum", http.StatusBadRequest)
|
||||
return "", 0, errors.New("missing checksum")
|
||||
}
|
||||
// Arbitrary minimum length, but we can fairly safely assume that all
|
||||
// hashing methods used produce a hash of at least 32 characters.
|
||||
if len(checksum) < 32 {
|
||||
http.Error(w, "checksum suspiciously short", http.StatusBadRequest)
|
||||
return "", 0, errors.New("checksum suspiciously short")
|
||||
}
|
||||
|
||||
filesizeStr, ok := vars["filesize"]
|
||||
if !ok {
|
||||
http.Error(w, "missing filesize", http.StatusBadRequest)
|
||||
return "", 0, errors.New("missing filesize")
|
||||
}
|
||||
filesize, err := strconv.ParseInt(filesizeStr, 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid filesize", http.StatusBadRequest)
|
||||
return "", 0, fmt.Errorf("invalid filesize: %v", err)
|
||||
}
|
||||
|
||||
return checksum, filesize, nil
|
||||
}
|
||||
@ -28,7 +28,7 @@ import (
|
||||
"strconv"
|
||||
|
||||
"git.blender.org/flamenco/pkg/shaman/config"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// Store represents the default Shaman file store.
|
||||
@ -40,8 +40,8 @@ type Store struct {
|
||||
}
|
||||
|
||||
// New returns a new file store.
|
||||
func New(conf config.Config) Storage {
|
||||
packageLogger.WithField("storageDir", conf.FileStorePath).Info("opening file store")
|
||||
func New(conf config.Config) *Store {
|
||||
log.Info().Str("storageDir", conf.FileStorePath).Msg("shaman: opening file store")
|
||||
store := &Store{
|
||||
conf.FileStorePath,
|
||||
storageBin{conf.FileStorePath, "uploading", true, ".tmp"},
|
||||
@ -55,24 +55,16 @@ func New(conf config.Config) Storage {
|
||||
func (s *Store) createDirectoryStructure() {
|
||||
mkdir := func(subdir string) {
|
||||
path := path.Join(s.baseDir, subdir)
|
||||
logger := packageLogger.WithField("path", path)
|
||||
stat, err := os.Stat(path)
|
||||
|
||||
if err == nil {
|
||||
if stat.IsDir() {
|
||||
// Exists and is a directory; nothing to do.
|
||||
logger := log.With().Str("path", path).Logger()
|
||||
logger.Debug().Msg("shaman: creating directory")
|
||||
|
||||
if err := os.MkdirAll(path, 0777); err != nil {
|
||||
if os.IsExist(err) {
|
||||
logger.Trace().Msg("shaman: directory exists")
|
||||
return
|
||||
}
|
||||
logger.Fatal("path exists but is not a directory")
|
||||
}
|
||||
|
||||
if !os.IsNotExist(err) {
|
||||
logger.WithError(err).Fatal("unable to stat directory")
|
||||
}
|
||||
|
||||
logger.Debug("creating directory")
|
||||
if err := os.MkdirAll(path, 0777); err != nil {
|
||||
logger.WithError(err).Fatal("unable to create directory")
|
||||
logger.Error().Err(err).Msg("shaman: unable to create directory")
|
||||
}
|
||||
}
|
||||
|
||||
@ -100,27 +92,29 @@ func (s *Store) partialFilePath(checksum string, filesize int64) string {
|
||||
func (s *Store) ResolveFile(checksum string, filesize int64, storedOnly StoredOnly) (path string, status FileStatus) {
|
||||
partial := s.partialFilePath(checksum, filesize)
|
||||
|
||||
logger := packageLogger.WithFields(logrus.Fields{
|
||||
"checksum": checksum,
|
||||
"filesize": filesize,
|
||||
"partialPath": partial,
|
||||
"storagePath": s.baseDir,
|
||||
})
|
||||
logger := log.With().
|
||||
Str("checksum", checksum).
|
||||
Int64("filesize", filesize).
|
||||
Str("partialPath", partial).
|
||||
Str("storagePath", s.baseDir).
|
||||
Logger()
|
||||
|
||||
if path = s.stored.resolve(partial); path != "" {
|
||||
// logger.WithField("path", path).Debug("found stored file")
|
||||
logger.Trace().Str("path", path).Msg("shaman: found stored file")
|
||||
return path, StatusStored
|
||||
}
|
||||
|
||||
if storedOnly != ResolveEverything {
|
||||
// logger.Debug("file does not exist in 'stored' state")
|
||||
logger.Trace().Msg("shaman: file does not exist in 'stored' state")
|
||||
return "", StatusDoesNotExist
|
||||
}
|
||||
|
||||
if path = s.uploading.resolve(partial); path != "" {
|
||||
logger.WithField("path", path).Debug("found currently uploading file")
|
||||
logger.Debug().Str("path", path).Msg("shaman: found currently uploading file")
|
||||
return path, StatusUploading
|
||||
}
|
||||
// logger.Debug("file does not exist")
|
||||
|
||||
logger.Trace().Msg("shaman: file does not exist")
|
||||
return "", StatusDoesNotExist
|
||||
}
|
||||
|
||||
@ -145,11 +139,10 @@ func (s *Store) MoveToStored(checksum string, filesize int64, uploadedFilePath s
|
||||
if err := os.MkdirAll(targetDir, 0777); err != nil {
|
||||
return err
|
||||
}
|
||||
logger := packageLogger.WithFields(logrus.Fields{
|
||||
"uploadedPath": uploadedFilePath,
|
||||
"storagePath": targetPath,
|
||||
})
|
||||
logger.Debug("moving uploaded file to storage")
|
||||
log.Debug().
|
||||
Str("uploadedPath", uploadedFilePath).
|
||||
Str("storagePath", targetPath).
|
||||
Msg("shaman: moving uploaded file to storage")
|
||||
if err := os.Rename(uploadedFilePath, targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -161,7 +154,7 @@ func (s *Store) MoveToStored(checksum string, filesize int64, uploadedFilePath s
|
||||
func (s *Store) removeFile(filePath string) error {
|
||||
err := os.Remove(filePath)
|
||||
if err != nil {
|
||||
packageLogger.WithError(err).Debug("unable to delete file; ignoring")
|
||||
log.Debug().Err(err).Msg("shaman: unable to delete file; ignoring")
|
||||
}
|
||||
|
||||
// Clean up directory structure, but ignore any errors (dirs may not be empty)
|
||||
@ -177,8 +170,8 @@ func (s *Store) removeFile(filePath string) error {
|
||||
func (s *Store) RemoveUploadedFile(filePath string) {
|
||||
// Check that the file path is actually in the 'uploading' storage.
|
||||
if !s.uploading.contains("", filePath) {
|
||||
packageLogger.WithField("file", filePath).Error(
|
||||
"filestore.Store.RemoveUploadedFile called with file not in 'uploading' storage bin")
|
||||
log.Error().Str("file", filePath).
|
||||
Msg("shaman: RemoveUploadedFile called with file not in 'uploading' storage bin")
|
||||
return
|
||||
}
|
||||
s.removeFile(filePath)
|
||||
@ -188,8 +181,8 @@ func (s *Store) RemoveUploadedFile(filePath string) {
|
||||
func (s *Store) RemoveStoredFile(filePath string) error {
|
||||
// Check that the file path is actually in the 'stored' storage.
|
||||
if !s.stored.contains("", filePath) {
|
||||
packageLogger.WithField("file", filePath).Error(
|
||||
"filestore.Store.RemoveStoredFile called with file not in 'stored' storage bin")
|
||||
log.Error().Str("file", filePath).
|
||||
Msg("shaman: RemoveStoredFile called with file not in 'stored' storage bin")
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return s.removeFile(filePath)
|
||||
|
||||
@ -1,29 +0,0 @@
|
||||
/* (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var packageLogger = logrus.WithField("package", "shaman/filestore")
|
||||
@ -43,13 +43,7 @@ func CreateTestStore() *Store {
|
||||
conf := config.Config{
|
||||
FileStorePath: tempDir,
|
||||
}
|
||||
storage := New(conf)
|
||||
store, ok := storage.(*Store)
|
||||
if !ok {
|
||||
panic("storage should be *Store")
|
||||
}
|
||||
|
||||
return store
|
||||
return New(conf)
|
||||
}
|
||||
|
||||
// CleanupTestStore deletes a store returned by CreateTestStore()
|
||||
|
||||
@ -1,68 +0,0 @@
|
||||
/* (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/kardianos/osext"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// RootPath returns the filename prefix to find bundled files.
|
||||
// Files are searched for relative to the current working directory as well as relative
|
||||
// to the currently running executable.
|
||||
func RootPath(fileToFind string) string {
|
||||
logger := packageLogger.WithField("fileToFind", fileToFind)
|
||||
|
||||
// Find as relative path, i.e. relative to CWD.
|
||||
_, err := os.Stat(fileToFind)
|
||||
if err == nil {
|
||||
logger.Debug("found in current working directory")
|
||||
return ""
|
||||
}
|
||||
|
||||
// Find relative to executable folder.
|
||||
exedirname, err := osext.ExecutableFolder()
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("unable to determine the executable's directory")
|
||||
return ""
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(exedirname, fileToFind)); os.IsNotExist(err) {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("unable to determine current working directory")
|
||||
}
|
||||
logger.WithFields(logrus.Fields{
|
||||
"cwd": cwd,
|
||||
"exedirname": exedirname,
|
||||
}).Error("unable to find file")
|
||||
return ""
|
||||
}
|
||||
|
||||
// Append a slash so that we can later just concatenate strings.
|
||||
logrus.WithField("exedirname", exedirname).Debug("found file")
|
||||
return exedirname + string(os.PathSeparator)
|
||||
}
|
||||
@ -1,87 +0,0 @@
|
||||
/* (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Errors returned by DecompressedReader
|
||||
var (
|
||||
ErrContentEncodingNotSupported = errors.New("Content-Encoding not supported")
|
||||
)
|
||||
|
||||
// wrapperCloserReader is a ReadCloser that closes both a wrapper and the wrapped reader.
|
||||
type wrapperCloserReader struct {
|
||||
wrapped io.ReadCloser
|
||||
wrapper io.ReadCloser
|
||||
}
|
||||
|
||||
func (cr *wrapperCloserReader) Close() error {
|
||||
errWrapped := cr.wrapped.Close()
|
||||
errWrapper := cr.wrapper.Close()
|
||||
|
||||
if errWrapped != nil {
|
||||
return errWrapped
|
||||
}
|
||||
return errWrapper
|
||||
}
|
||||
|
||||
func (cr *wrapperCloserReader) Read(p []byte) (n int, err error) {
|
||||
return cr.wrapper.Read(p)
|
||||
}
|
||||
|
||||
// DecompressedReader returns a reader that decompresses the body.
|
||||
// The compression scheme is determined by the Content-Encoding header.
|
||||
// Closing the returned reader is the caller's responsibility.
|
||||
func DecompressedReader(request *http.Request) (io.ReadCloser, error) {
|
||||
var wrapper io.ReadCloser
|
||||
var err error
|
||||
|
||||
switch request.Header.Get("Content-Encoding") {
|
||||
case "gzip":
|
||||
wrapper, err = gzip.NewReader(request.Body)
|
||||
case "identity", "":
|
||||
return request.Body, nil
|
||||
default:
|
||||
return nil, ErrContentEncodingNotSupported
|
||||
}
|
||||
|
||||
return &wrapperCloserReader{
|
||||
wrapped: request.Body,
|
||||
wrapper: wrapper,
|
||||
}, err
|
||||
}
|
||||
|
||||
// CompressBuffer GZip-compresses the payload into a buffer, and returns it.
|
||||
func CompressBuffer(payload []byte) *bytes.Buffer {
|
||||
var bodyBuf bytes.Buffer
|
||||
compressor := gzip.NewWriter(&bodyBuf)
|
||||
compressor.Write(payload)
|
||||
compressor.Close()
|
||||
return &bodyBuf
|
||||
}
|
||||
@ -1,29 +0,0 @@
|
||||
/* (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var packageLogger = logrus.WithField("package", "shaman/httpserver")
|
||||
@ -1,72 +0,0 @@
|
||||
/* (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"git.blender.org/flamenco/pkg/shaman/jwtauth"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var userInfo = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
tokenSubject, ok := jwtauth.SubjectFromContext(r.Context())
|
||||
if !ok {
|
||||
fmt.Fprintf(w, "You are unknown to me")
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, "You are subject %s", tokenSubject)
|
||||
})
|
||||
|
||||
// RegisterTestRoutes registers some routes that should only be used for testing.
|
||||
func RegisterTestRoutes(r *mux.Router, auther jwtauth.Authenticator) {
|
||||
// On the default page we will simply serve our static index page.
|
||||
r.Handle("/", http.FileServer(http.Dir("./views/")))
|
||||
|
||||
// We will setup our server so we can serve static assest like images, css from the /static/{file} route
|
||||
r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir("./static/"))))
|
||||
|
||||
getTokenHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
tokenString, err := auther.GenerateToken()
|
||||
if err != nil {
|
||||
logger := packageLogger.WithFields(logrus.Fields{
|
||||
logrus.ErrorKey: err,
|
||||
"remoteAddr": r.RemoteAddr,
|
||||
"requestURI": r.RequestURI,
|
||||
"requestMethod": r.Method,
|
||||
})
|
||||
logger.Warning("unable to sign JWT")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(fmt.Sprintf("error signing token: %v", err)))
|
||||
return
|
||||
}
|
||||
|
||||
w.Write([]byte(tokenString))
|
||||
})
|
||||
|
||||
r.Handle("/get-token", getTokenHandler).Methods("GET")
|
||||
r.Handle("/my-info", auther.Wrap(userInfo)).Methods("GET")
|
||||
}
|
||||
@ -9,3 +9,7 @@ package jwtauth
|
||||
|
||||
type Authenticator interface {
|
||||
}
|
||||
|
||||
type AlwaysDeny struct{}
|
||||
|
||||
var _ Authenticator = (*AlwaysDeny)(nil)
|
||||
|
||||
@ -1,29 +0,0 @@
|
||||
/* (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package shaman
|
||||
|
||||
import (
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var packageLogger = logrus.WithField("package", "shaman")
|
||||
@ -23,15 +23,17 @@
|
||||
package shaman
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"git.blender.org/flamenco/pkg/api"
|
||||
"git.blender.org/flamenco/pkg/shaman/checkout"
|
||||
"git.blender.org/flamenco/pkg/shaman/config"
|
||||
"git.blender.org/flamenco/pkg/shaman/fileserver"
|
||||
"git.blender.org/flamenco/pkg/shaman/filestore"
|
||||
"git.blender.org/flamenco/pkg/shaman/httpserver"
|
||||
"git.blender.org/flamenco/pkg/shaman/jwtauth"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// Server represents a Shaman Server.
|
||||
@ -49,9 +51,8 @@ type Server struct {
|
||||
|
||||
// NewServer creates a new Shaman server.
|
||||
func NewServer(conf config.Config, auther jwtauth.Authenticator) *Server {
|
||||
|
||||
if !conf.Enabled {
|
||||
packageLogger.Warning("Shaman server is disabled")
|
||||
log.Info().Msg("Shaman server is disabled")
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -60,14 +61,14 @@ func NewServer(conf config.Config, auther jwtauth.Authenticator) *Server {
|
||||
fileServer := fileserver.New(fileStore)
|
||||
|
||||
shamanServer := &Server{
|
||||
conf,
|
||||
auther,
|
||||
fileStore,
|
||||
fileServer,
|
||||
checkoutMan,
|
||||
config: conf,
|
||||
auther: auther,
|
||||
fileStore: fileStore,
|
||||
fileServer: fileServer,
|
||||
checkoutMan: checkoutMan,
|
||||
|
||||
make(chan struct{}),
|
||||
sync.WaitGroup{},
|
||||
shutdownChan: make(chan struct{}),
|
||||
wg: sync.WaitGroup{},
|
||||
}
|
||||
|
||||
return shamanServer
|
||||
@ -76,32 +77,70 @@ func NewServer(conf config.Config, auther jwtauth.Authenticator) *Server {
|
||||
// Go starts goroutines for background operations.
|
||||
// After Go() has been called, use Close() to stop those goroutines.
|
||||
func (s *Server) Go() {
|
||||
packageLogger.Info("Shaman server starting")
|
||||
log.Info().Msg("Shaman server starting")
|
||||
s.fileServer.Go()
|
||||
|
||||
if s.config.GarbageCollect.Period == 0 {
|
||||
packageLogger.Warning("garbage collection disabled, set garbageCollect.period > 0 in configuration")
|
||||
log.Warn().Msg("garbage collection disabled, set garbageCollect.period > 0 in configuration")
|
||||
} else if s.config.GarbageCollect.SilentlyDisable {
|
||||
packageLogger.Debug("not starting garbage collection")
|
||||
log.Debug().Msg("not starting garbage collection")
|
||||
} else {
|
||||
s.wg.Add(1)
|
||||
go s.periodicCleanup()
|
||||
}
|
||||
}
|
||||
|
||||
// AddRoutes adds the Shaman server endpoints to the given router.
|
||||
func (s *Server) AddRoutes(router *mux.Router) {
|
||||
s.checkoutMan.AddRoutes(router, s.auther)
|
||||
s.fileServer.AddRoutes(router, s.auther)
|
||||
|
||||
httpserver.RegisterTestRoutes(router, s.auther)
|
||||
}
|
||||
|
||||
// Close shuts down the Shaman server.
|
||||
func (s *Server) Close() {
|
||||
packageLogger.Info("shutting down Shaman server")
|
||||
log.Info().Msg("shutting down Shaman server")
|
||||
|
||||
close(s.shutdownChan)
|
||||
|
||||
s.fileServer.Close()
|
||||
s.checkoutMan.Close()
|
||||
s.wg.Wait()
|
||||
}
|
||||
|
||||
// Checkout creates a directory, and symlinks the required files into it. The
|
||||
// files must all have been uploaded to Shaman before calling this.
|
||||
func (s *Server) Checkout(ctx context.Context, checkoutID string, checkout api.ShamanCheckout) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Requirements checks a Shaman Requirements file, and returns the subset
|
||||
// containing the unknown files.
|
||||
func (s *Server) Requirements(ctx context.Context, requirements api.ShamanRequirements) (api.ShamanRequirements, error) {
|
||||
return requirements, nil
|
||||
}
|
||||
|
||||
var fsStatusToApiStatus = map[filestore.FileStatus]api.ShamanFileStatusStatus{
|
||||
filestore.StatusDoesNotExist: api.ShamanFileStatusStatusUnknown,
|
||||
filestore.StatusUploading: api.ShamanFileStatusStatusUploading,
|
||||
filestore.StatusStored: api.ShamanFileStatusStatusStored,
|
||||
}
|
||||
|
||||
// Check the status of a file on the Shaman server.
|
||||
// status (stored, currently being uploaded, unknown).
|
||||
func (s *Server) FileStoreCheck(ctx context.Context, checksum string, filesize int64) api.ShamanFileStatusStatus {
|
||||
status := s.fileServer.CheckFile(checksum, filesize)
|
||||
apiStatus, ok := fsStatusToApiStatus[status]
|
||||
if !ok {
|
||||
log.Warn().
|
||||
Str("checksum", checksum).
|
||||
Int64("filesize", filesize).
|
||||
Int("fileserverStatus", int(status)).
|
||||
Msg("shaman: unknown status on fileserver")
|
||||
return api.ShamanFileStatusStatusUnknown
|
||||
}
|
||||
return apiStatus
|
||||
}
|
||||
|
||||
// Store a new file on the Shaman server. Note that the Shaman server can return
|
||||
// early when another client finishes uploading the exact same file, to prevent
|
||||
// double uploads.
|
||||
func (s *Server) FileStore(ctx context.Context, file io.ReadCloser, checksum string, filesize int64, canDefer bool, originalFilename string) error {
|
||||
err := s.fileServer.ReceiveFile(ctx, file, checksum, filesize, canDefer)
|
||||
// TODO: Maybe translate this error into something that can be understood by
|
||||
// the caller without relying on types declared in the `fileserver` package?
|
||||
return err
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user