More status change acks & checks to get stable flow between worker states
This commit is contained in:
Sybren A. Stüvel
parent
c4df62d5d4
commit
270c54fdb7
@ -3,6 +3,7 @@ package api_impl
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
||||
@ -53,16 +54,21 @@ func newMockedFlamenco(mockCtrl *gomock.Controller) mockedFlamenco {
|
||||
}
|
||||
|
||||
func (mf *mockedFlamenco) prepareMockedJSONRequest(worker *persistence.Worker, requestBody interface{}) echo.Context {
|
||||
|
||||
e := echo.New()
|
||||
|
||||
bodyBytes, err := json.MarshalIndent(requestBody, "", " ")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", bytes.NewBuffer(bodyBytes))
|
||||
req.Header.Add(echo.HeaderContentType, "application/json")
|
||||
c := mf.prepareMockedRequest(worker, bytes.NewBuffer(bodyBytes))
|
||||
c.Request().Header.Add(echo.HeaderContentType, "application/json")
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (mf *mockedFlamenco) prepareMockedRequest(worker *persistence.Worker, body io.Reader) echo.Context {
|
||||
e := echo.New()
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", body)
|
||||
rec := httptest.NewRecorder()
|
||||
c := e.NewContext(req, rec)
|
||||
requestWorkerStore(c, worker)
|
||||
|
||||
@ -21,6 +21,7 @@ package api_impl
|
||||
* ***** END GPL LICENSE BLOCK ***** */
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
@ -143,6 +144,10 @@ func (f *Flamenco) SignOff(e echo.Context) error {
|
||||
logger.Info().Msg("worker signing off")
|
||||
w := requestWorkerOrPanic(e)
|
||||
w.Status = api.WorkerStatusOffline
|
||||
if w.StatusRequested == api.WorkerStatusShutdown {
|
||||
w.StatusRequested = ""
|
||||
}
|
||||
|
||||
// TODO: check whether we should pass the request context here, or a generic
|
||||
// background context, as this should be stored even when the HTTP connection
|
||||
// is aborted.
|
||||
@ -183,10 +188,24 @@ func (f *Flamenco) WorkerStateChanged(e echo.Context) error {
|
||||
return sendAPIError(e, http.StatusBadRequest, "invalid format")
|
||||
}
|
||||
|
||||
logger.Info().Str("newStatus", string(req.Status)).Msg("worker changed status")
|
||||
|
||||
w := requestWorkerOrPanic(e)
|
||||
logger = logger.With().
|
||||
Str("currentStatus", string(w.Status)).
|
||||
Str("newStatus", string(req.Status)).
|
||||
Logger()
|
||||
|
||||
w.Status = req.Status
|
||||
if w.StatusRequested != "" && req.Status != w.StatusRequested {
|
||||
logger.Warn().
|
||||
Str("workersRequestedStatus", string(w.StatusRequested)).
|
||||
Msg("worker changed to status that was not requested")
|
||||
} else {
|
||||
logger.Info().Msg("worker changed status")
|
||||
// Either there was no status change request (and this is a no-op) or the
|
||||
// status change was actually acknowledging the request.
|
||||
w.StatusRequested = ""
|
||||
}
|
||||
|
||||
err = f.persist.SaveWorkerStatus(e.Request().Context(), w)
|
||||
if err != nil {
|
||||
logger.Warn().Err(err).
|
||||
@ -200,13 +219,27 @@ func (f *Flamenco) WorkerStateChanged(e echo.Context) error {
|
||||
|
||||
func (f *Flamenco) ScheduleTask(e echo.Context) error {
|
||||
logger := requestLogger(e)
|
||||
worker := requestWorkerOrPanic(e)
|
||||
logger.Info().Msg("worker requesting task")
|
||||
|
||||
// Figure out which worker is requesting a task:
|
||||
worker := requestWorker(e)
|
||||
if worker == nil {
|
||||
logger.Warn().Msg("task requested by non-worker")
|
||||
return sendAPIError(e, http.StatusBadRequest, "not authenticated as Worker")
|
||||
// Check that this worker is actually allowed to do work.
|
||||
requiredStatusToGetTask := api.WorkerStatusAwake
|
||||
if worker.Status != api.WorkerStatusAwake {
|
||||
logger.Warn().
|
||||
Str("workerStatus", string(worker.Status)).
|
||||
Str("requiredStatus", string(requiredStatusToGetTask)).
|
||||
Msg("worker asking for task but is in wrong state")
|
||||
return sendAPIError(e, http.StatusConflict,
|
||||
fmt.Sprintf("worker is in state %q, requires state %q to execute tasks", worker.Status, requiredStatusToGetTask))
|
||||
}
|
||||
if worker.StatusRequested != "" {
|
||||
logger.Warn().
|
||||
Str("workerStatus", string(worker.Status)).
|
||||
Str("requestedStatus", string(worker.StatusRequested)).
|
||||
Msg("worker asking for task but needs state change first")
|
||||
return e.JSON(http.StatusLocked, api.WorkerStateChange{
|
||||
StatusRequested: worker.StatusRequested,
|
||||
})
|
||||
}
|
||||
|
||||
// Get a task to execute:
|
||||
|
||||
102
internal/manager/api_impl/workers_test.go
Normal file
102
internal/manager/api_impl/workers_test.go
Normal file
@ -0,0 +1,102 @@
|
||||
package api_impl
|
||||
|
||||
/* ***** BEGIN GPL LICENSE BLOCK *****
|
||||
*
|
||||
* Original Code Copyright (C) 2022 Blender Foundation.
|
||||
*
|
||||
* This file is part of Flamenco.
|
||||
*
|
||||
* Flamenco is free software: you can redistribute it and/or modify it under
|
||||
* the terms of the GNU General Public License as published by the Free Software
|
||||
* Foundation, either version 3 of the License, or (at your option) any later
|
||||
* version.
|
||||
*
|
||||
* Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
* A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* Flamenco. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
* ***** END GPL LICENSE BLOCK ***** */
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"gitlab.com/blender/flamenco-ng-poc/internal/manager/persistence"
|
||||
"gitlab.com/blender/flamenco-ng-poc/pkg/api"
|
||||
)
|
||||
|
||||
func TestTaskScheduleHappy(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
|
||||
mf := newMockedFlamenco(mockCtrl)
|
||||
worker := testWorker()
|
||||
|
||||
// Expect a call into the persistence layer, which should return a scheduled task.
|
||||
job := persistence.Job{
|
||||
UUID: "583a7d59-887a-4c6c-b3e4-a753018f71b0",
|
||||
}
|
||||
task := persistence.Task{
|
||||
UUID: "4107c7aa-e86d-4244-858b-6c4fce2af503",
|
||||
Job: &job,
|
||||
}
|
||||
mf.persistence.EXPECT().ScheduleTask(&worker).Return(&task, nil)
|
||||
|
||||
echoCtx := mf.prepareMockedRequest(&worker, nil)
|
||||
err := mf.flamenco.ScheduleTask(echoCtx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
resp := echoCtx.Response().Writer.(*httptest.ResponseRecorder)
|
||||
assert.Equal(t, http.StatusOK, resp.Code)
|
||||
// TODO: check that the returned JSON actually matches what we expect.
|
||||
}
|
||||
|
||||
func TestTaskScheduleNonActiveStatus(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
|
||||
mf := newMockedFlamenco(mockCtrl)
|
||||
worker := testWorker()
|
||||
worker.Status = api.WorkerStatusAsleep
|
||||
|
||||
// Explicitly NO expected calls to the persistence layer. Since the worker is
|
||||
// not in a state that allows task execution, there should be no DB queries.
|
||||
|
||||
echoCtx := mf.prepareMockedRequest(&worker, nil)
|
||||
err := mf.flamenco.ScheduleTask(echoCtx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
resp := echoCtx.Response().Writer.(*httptest.ResponseRecorder)
|
||||
assert.Equal(t, http.StatusConflict, resp.Code)
|
||||
}
|
||||
|
||||
func TestTaskScheduleOtherStatusRequested(t *testing.T) {
|
||||
mockCtrl := gomock.NewController(t)
|
||||
defer mockCtrl.Finish()
|
||||
|
||||
mf := newMockedFlamenco(mockCtrl)
|
||||
worker := testWorker()
|
||||
worker.StatusRequested = api.WorkerStatusAsleep
|
||||
|
||||
// Explicitly NO expected calls to the persistence layer. Since the worker is
|
||||
// not in a state that allows task execution, there should be no DB queries.
|
||||
|
||||
echoCtx := mf.prepareMockedRequest(&worker, nil)
|
||||
err := mf.flamenco.ScheduleTask(echoCtx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
resp := echoCtx.Response().Writer.(*httptest.ResponseRecorder)
|
||||
assert.Equal(t, http.StatusLocked, resp.Code)
|
||||
|
||||
responseBody := api.WorkerStateChange{}
|
||||
err = json.Unmarshal(resp.Body.Bytes(), &responseBody)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, worker.StatusRequested, responseBody.StatusRequested)
|
||||
}
|
||||
@ -158,11 +158,6 @@ func TestTwoJobsThreeTasks(t *testing.T) {
|
||||
assert.Equal(t, att2_3.Name, task.Name, "the 3rd task of the 2nd job should have been chosen")
|
||||
}
|
||||
|
||||
// To test: worker with non-active state.
|
||||
// Unlike Flamenco v2, this Manager shouldn't change a worker's status
|
||||
// simply because it requests a task. New tasks for non-awake workers
|
||||
// should be rejected.
|
||||
|
||||
// To test: blacklists
|
||||
|
||||
// To test: variable replacement
|
||||
|
||||
@ -67,7 +67,14 @@ func (db *DB) FetchWorker(ctx context.Context, uuid string) (*Worker, error) {
|
||||
}
|
||||
|
||||
func (db *DB) SaveWorkerStatus(ctx context.Context, w *Worker) error {
|
||||
if err := db.gormDB.Model(w).Select("status").Updates(Worker{Status: w.Status}).Error; err != nil {
|
||||
err := db.gormDB.
|
||||
Model(w).
|
||||
Select("status", "status_requested").
|
||||
Updates(Worker{
|
||||
Status: w.Status,
|
||||
StatusRequested: w.StatusRequested,
|
||||
}).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("error saving worker: %w", err)
|
||||
}
|
||||
return nil
|
||||
|
||||
@ -22,7 +22,6 @@ package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
@ -37,7 +36,7 @@ func (w *Worker) gotoStateAsleep(ctx context.Context) {
|
||||
|
||||
w.state = api.WorkerStatusAsleep
|
||||
w.doneWg.Add(2)
|
||||
go w.ackStateChange(ctx, w.state)
|
||||
w.ackStateChange(ctx, w.state)
|
||||
go w.runStateAsleep(ctx)
|
||||
}
|
||||
|
||||
@ -49,37 +48,18 @@ func (w *Worker) runStateAsleep(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Debug().Msg("state fetching interrupted by context cancellation")
|
||||
logger.Debug().Msg("asleep state interrupted by context cancellation")
|
||||
return
|
||||
case <-w.doneChan:
|
||||
logger.Debug().Msg("state fetching interrupted by shutdown")
|
||||
logger.Debug().Msg("asleep state interrupted by shutdown")
|
||||
return
|
||||
case <-time.After(durationSleepCheck):
|
||||
}
|
||||
if !w.isState(api.WorkerStatusAsleep) {
|
||||
logger.Debug().Str("newStatus", string(w.state)).Msg("state fetching interrupted by state change")
|
||||
return
|
||||
}
|
||||
|
||||
resp, err := w.client.WorkerStateWithResponse(ctx)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("error checking upstream state changes")
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case resp.JSON200 != nil:
|
||||
log.Info().
|
||||
Str("requestedStatus", string(resp.JSON200.StatusRequested)).
|
||||
Msg("Manager requests status change")
|
||||
w.changeState(ctx, resp.JSON200.StatusRequested)
|
||||
return
|
||||
case resp.StatusCode() == http.StatusNoContent:
|
||||
log.Debug().Msg("we can keep sleeping")
|
||||
default:
|
||||
log.Warn().
|
||||
Int("code", resp.StatusCode()).
|
||||
Str("error", string(resp.Body)).
|
||||
Msg("unable to obtain requested state for unknown reason")
|
||||
newStatus := w.queryManagerForStateChange(ctx)
|
||||
if newStatus != nil {
|
||||
logger.Debug().Str("newStatus", string(*newStatus)).Msg("asleep state interrupted by state change")
|
||||
w.changeState(ctx, *newStatus)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -42,7 +42,7 @@ func (w *Worker) gotoStateAwake(ctx context.Context) {
|
||||
w.state = api.WorkerStatusAwake
|
||||
|
||||
w.doneWg.Add(2)
|
||||
go w.ackStateChange(ctx, w.state)
|
||||
w.ackStateChange(ctx, w.state)
|
||||
go w.runStateAwake(ctx)
|
||||
}
|
||||
|
||||
@ -62,6 +62,9 @@ func (w *Worker) runStateAwake(ctx context.Context) {
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Interface("task", *task).Msg("error executing task")
|
||||
}
|
||||
|
||||
// Do some rate limiting. This is mostly useful while developing.
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
@ -84,10 +87,6 @@ func (w *Worker) fetchTask(ctx context.Context) *api.AssignedTask {
|
||||
return nil
|
||||
case <-time.After(wait):
|
||||
}
|
||||
if !w.isState(api.WorkerStatusAwake) {
|
||||
logger.Debug().Msg("task fetching interrupted by state change")
|
||||
return nil
|
||||
}
|
||||
|
||||
resp, err := w.client.ScheduleTaskWithResponse(ctx)
|
||||
if err != nil {
|
||||
@ -112,18 +111,16 @@ func (w *Worker) fetchTask(ctx context.Context) *api.AssignedTask {
|
||||
Str("error", string(resp.JSON403.Message)).
|
||||
Msg("access denied")
|
||||
wait = durationFetchFailed
|
||||
continue
|
||||
case resp.StatusCode() == http.StatusNoContent:
|
||||
log.Info().Msg("no task available")
|
||||
wait = durationNoTask
|
||||
continue
|
||||
default:
|
||||
log.Warn().
|
||||
Int("code", resp.StatusCode()).
|
||||
Str("error", string(resp.Body)).
|
||||
Msg("unable to obtain task for unknown reason")
|
||||
wait = durationFetchFailed
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
56
internal/worker/statemonitor.go
Normal file
56
internal/worker/statemonitor.go
Normal file
@ -0,0 +1,56 @@
|
||||
package worker
|
||||
|
||||
/* ***** BEGIN GPL LICENSE BLOCK *****
|
||||
*
|
||||
* Original Code Copyright (C) 2022 Blender Foundation.
|
||||
*
|
||||
* This file is part of Flamenco.
|
||||
*
|
||||
* Flamenco is free software: you can redistribute it and/or modify it under
|
||||
* the terms of the GNU General Public License as published by the Free Software
|
||||
* Foundation, either version 3 of the License, or (at your option) any later
|
||||
* version.
|
||||
*
|
||||
* Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
* A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* Flamenco. If not, see <https://www.gnu.org/licenses/>.
|
||||
*
|
||||
* ***** END GPL LICENSE BLOCK ***** */
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"gitlab.com/blender/flamenco-ng-poc/pkg/api"
|
||||
)
|
||||
|
||||
// queryManagerForStateChange asks the Manager whether we should go to another state or not.
|
||||
// Any error communicating with the Manager is logged but otherwise ignored.
|
||||
// Returns nil when no state change is requested.
|
||||
func (w *Worker) queryManagerForStateChange(ctx context.Context) *api.WorkerStatus {
|
||||
resp, err := w.client.WorkerStateWithResponse(ctx)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("error checking upstream state changes")
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case resp.JSON200 != nil:
|
||||
log.Info().
|
||||
Str("requestedStatus", string(resp.JSON200.StatusRequested)).
|
||||
Msg("Manager requests status change")
|
||||
return &resp.JSON200.StatusRequested
|
||||
case resp.StatusCode() == http.StatusNoContent:
|
||||
log.Debug().Msg("we can stay in the current state")
|
||||
default:
|
||||
log.Warn().
|
||||
Int("code", resp.StatusCode()).
|
||||
Str("error", string(resp.Body)).
|
||||
Msg("unable to obtain requested state for unknown reason")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -150,6 +150,8 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema: {$ref: "#/components/schemas/SecurityError"}
|
||||
"409":
|
||||
description: Worker is not in the active state, so is not allowed to execute tasks right now.
|
||||
"423":
|
||||
description: Worker cannot obtain new tasks, but must go to another state.
|
||||
content:
|
||||
|
||||
@ -18,62 +18,62 @@ import (
|
||||
// Base64 encoded, gzipped, json marshaled Swagger object
|
||||
var swaggerSpec = []string{
|
||||
|
||||
"H4sIAAAAAAAC/9xb624bxxV+lcGmQBJ0ScqWW6D8VceOHRm+CKGM/IgFarh7yB1pdmYzM0uaNQTkIfom",
|
||||
"bYD+aH71BZQ3Ks5c9k5RiiU3aRAEK+7MmXP9zmUnH6JE5oUUIIyOph8inWSQU/v4WGu2EpCeUH2Bf6eg",
|
||||
"E8UKw6SIpq23hGlCicEnqgkz+LeCBNgaUrLYEpMB+U6qC1DjKI4KJQtQhoE9JZF5TkVqn5mB3D78QcEy",
|
||||
"mkafTWrmJp6zyRO3IbqMI7MtIJpGVCm6xb/P5QJ3+5+1UUys/O/zQjGpmNk2FjBhYAUqrHC/DmwXNB9+",
|
||||
"cT1Nbagp94qD+pu5lSgR1Re7GSlLluKLpVQ5NdHU/RB3F17GkYIfSqYgjabfh0WoHC9LxVtDhI6WGipp",
|
||||
"chXX9jqtzpWLc0gMMvh4TRmnCw4v5GIGxiA7Pc+ZMbHiQLR7T+SSUPJCLghS0wMOkkmWuMc2ne8yEGTF",
|
||||
"1iBiwlnOjPWzNeUsxf+WoImR+JsG4omMyRvBt6TUyCPZMJMRpzR7OJ5duWBP+V1nS2FJS276fJ1kQPxL",
|
||||
"xwfRmdwIzwwpNSiyQd5TMKByJuz5GdNBJWMkDykzyKWj749aUq4h7uvBZKCQPuVcbghu7dIkdGlwTQbk",
|
||||
"XC5IRjVZAAiiy0XOjIF0TL6TJU8Jywu+JSlwcNs4J/CeaUeQ6gtNllI50udyERMqUox1mReM4xpmxu9E",
|
||||
"7ZMLKTlQgRJdwLavrKMUhGFLBsrTrRwjJnmpDVkAKQX7oXTmYqISIVisZ6ja92+hOZbnkDJqgG+JAvRn",
|
||||
"Qu0xKSyZYLghRle1guORseVHlsb9VFBlWFJyqior7lCDLhchwK/DhYFQmvmdlTPemsKJ375mmnV9y6jy",
|
||||
"OgWhD7c9ytvi7ZELYVRW8CZFvuDsAgglX3EQKShC03QkxZdjMgOD5M6sQc5cILiMQQVB9FSC8uoMk1GD",
|
||||
"R5c8FZ9bZ6hiCURqY0kPK7oDguh8ftENgWtW26mDX+VihG+cOzhnDDYnT0qlQBi+JRKRhga61rsbWKPH",
|
||||
"5Oybx7Nvvn46f3b08uv58eOTb85cHk2ZgsRItSUFNRn5Izl7F00+s/+8i84ILQpUaerEBlHmKN+ScZjj",
|
||||
"+iiOUqbCo/3ZY35GdQbpvF55OhA8u5ymj3JeAw3pGxHrAJZqcvT02KH51oqNTuNdYkxeSyJAG0hRMWVi",
|
||||
"SgWafGEBVsckZQkeRRUD/SWhCogui0Iq0xXdMx9j7j18iEJzSU0UW1/YK+SwdCEf1We6OoZp8ooKugLl",
|
||||
"kI8ZG/o0RygfSF6cLoDfrqjwyrx5QTSUdHv5qhMO3iUce40z98UGamsgFb9k2gRnsN69W299HYVC49dJ",
|
||||
"fNJCxB3i1kcMCRgqyp5Y/gVRUCjQyAKhRLvyxddBFoneQ1Ia2Ffp3sjiHeaGzXatub5WSiok1a2zU2jV",
|
||||
"jiFa+oVrDlrT1RCvHXYszXr9EDcvXElOOX+zjKbfX2/XWShGcNdl3BNBATUwZCd8waQghuWgDc0LRIEg",
|
||||
"aEoNjPDNULHABsi9fXv0NID7C1s17ym4b1rrY4BWpX5ZpHcsTcc6ltOgs/q8itnTy1NnoFdgaEoNtYZK",
|
||||
"U1vsUH7c0n1P4k43qBbMKKq2JPfEfLLTY/JKKhsuBYf3TaRPqMBckUssNi1OlBhb5IyOF+PkjAhpnB5C",
|
||||
"YXgBW4wqeE+Rlndx62jTaFYoZoA8U2yVIfZjZTCGnDKOXG8XCsRfFz7xSLUKK1xMRjO7gMzMf/69Bt6A",
|
||||
"k5YjzxpxOqwnV0MN7q0cJKQtmhi2th0VFQlqwDVXBQfjn4VTFpNitKTMrageClpq+/BDCaV9oCrJsOOu",
|
||||
"Hl1WdORH6Bk22XoirR/ss6NSoopGzcOjONpQ21GMllKNsH7Qg2n1W1gxbUBB6iCwD0I0TRXoYYfiVJu5",
|
||||
"VUq7o26kTJZc7O7FOTUYJMMIK5dmQ9UO+L1R7DqR6vCtEty86o7bCWxvA/lR3Xyli7hSarOrD8qIo8QV",
|
||||
"pJbLqKvlhmZ2SDSE6TNISsXMdkemuXH6uC5vtFLBYHlWN2Z1E4vZ+BmnOYhEdqAib4Dc/cGGf3F49Xfy",
|
||||
"y49XP139fPXPq59++fHqX1c/X/2jOW6Z/umgnfj9KfMkT6Np9MH/eYkWzEpxMdfsbxBND1Emo2hi5rRM",
|
||||
"mQyQg0Fpa/ppNFF250QvJ+dygQ4MAh48PBxbks1Ucvz6Of5Z6Gj68FEcLbGM1dE0ejB6cIDldE5XoOdS",
|
||||
"zdcsBYk1gv0liiNZmqI0rpWA9waEdnYZFxZyHAdzt6rNkjukYqoRF5qhqUZe8JHb4qZsbe+q7bgn11Z5",
|
||||
"7aYzvKoXRuMMDPQa5tqX5sPSRq9+fTD4YPZTtoqrodhojAxvkU+qzFFBPcZ+nVlukid80hkCf2TqrS0x",
|
||||
"BnrF6h2x8wNhMNtTXylj0LrixI1/rGTkXXlw8PDPhMuVdvMFO15m5nPt6207KOt6RzN/tHl4I2DEmfDT",
|
||||
"HpGyBA/cZBQpJlXXntn2GssQOx1EhvDgMXmzBrVBsNCkULBmstR862QJh1Ylz1CFyOXAKPSlXBFkqjFU",
|
||||
"8zjd3h9HG8Y5Vkuh+0cprG4sB0AVZ9hzTEXJuZ8jz249fx6qe5yNXGpX1PHdRfyPSMyQKDDDrz4ywXbi",
|
||||
"y5/Uyo2DRzRy6+lOfczYSry5rSZCrp2vQWmvyPsWu1En7JC2x9U1Uhtq4ElGxQr6oru4nddwcauCqmut",
|
||||
"LrEbMZXu4uoOeNnDQRuLtaHKuMClG3phqzTNAbCTA1s1xZHOSpPKjZ1dgvar5XKJeDCAsC5YbN01Q66d",
|
||||
"eBvLwJyWmPp7fawGhbZH0EUgc4vJ0dOYFFTrjVRpeOWiw31NIdSEpaoR9ohqVl92zEo1S2qYyowpokvk",
|
||||
"kYmldOMGYWhi6rlHFOoycgIUg69U3O/U08lkGao2Jif99vJbN0R+RlVOcjdHIo+Pj7CeZQkIDY1znh+/",
|
||||
"XB/26G82m/FKlFjETfwePVkVfHQ4PhiDGGcmd30fM7zFrT8uiqMqaqMH44PxAa6WBQhaMKz47E+YIU1m",
|
||||
"LTOhBbMFmPVJqa0q0DOtMo9SN0jOmXETBu/pX8l0G9QHwu6hRcExWTEpJufaoYbz231e3R6nXPa0aoec",
|
||||
"0lfPUdPpsai0UaALiZrCkx4eHHxSzjZUE10mCehlyfmWuE9MkBImfAZfs7Sk3H2VGnc+yd0Jm67BGeDP",
|
||||
"viChf7GxWeY5VdvKqoQSARs7EMXUXrmTn4I2xoa2WqBYVdo5pY5OW+RehM8qGp2PgEgLyYSx8lY+NqnS",
|
||||
"xAoGHO05mGp2e49W7Q+KB1RXLaqHxR0FPgdDeG+gbGetGTDVmbdfo7r6qEr95/V35pb+PpzLxZyllztV",
|
||||
"+AxMkrlQrc+3A02GUvnPLR6CHLFeRMUNPe5r+k/v0U7XBJ2F77Y5rOT2BaEL973T2u4Gfus2idSDaI6c",
|
||||
"B7W7DDNRfmY02tQjo0GwDMMlP1q6H8QcKHYHFFW3LYH7TwqevTHbAIsC3YuTwMMnBcdSwPsCEgMpAb+m",
|
||||
"6RiBfY+Qm2DP4Ev+h9OBTc4kiAv1Tt31KM1WYiSXy2vyLhbvy2UfCh/1a6jfniJ9EWixp1X+fX+KqFHr",
|
||||
"7BVVF826j2Ij7crLPdp+QrmfyDsPsxdpOLjQDxnsQtgLAbD9XAFZSXeVx5IfD5tE7LGIuNeg9kfsDudq",
|
||||
"sPQpY7nfV/0ugvnGPvi4NBkI44YtfqSD3hAuj2yqb6V37JAKaLrFVUjPfatvjZlYbfC+uxo/xRqsBBom",
|
||||
"i/7XnmE5JYl9T+pm+TLeBWZk947ftkvd3j2SDJILsgk3mDJQ4G4ZbXcoYdgPRkljtDAIXgNjiHsFsuZB",
|
||||
"A+p9XaVGJ+cN8Oz/K+95PPd2c0oYk5OMaZLYK44LezOJJggYHFJXmLohs8eSeujd8pWYSIXIFbQS8AXU",
|
||||
"iMuEcgttlOu7xrM1tKQpdc9Vjb+jvSO9JhmkJYcTN1u+vwaweWN8wLD2rniz890FVK+lv2vavj9nB+Th",
|
||||
"es1lHD06OLy7kUTro+YA88egQhP+FARzoPno4eGnRfzg3FQIaYhcGMqErYatvmKyKI27hbeS9kKwkBb+",
|
||||
"XBDcMpDeOOq0ot+w3T4Pt6bW3u/UwNii4biTD3Yg7dvvYRdufF66SQfuCX58C373KN6QZFeI+DIF22dk",
|
||||
"0d3B+BUgfpJBoLWxiJdAERLdo4O/DG8w4X/o8MHcdCNntJhof5W9po3e2KL/e8kWb+svjyh5TMy2YAnl",
|
||||
"fNv6UFgouVKgdexvMvkL4YosKeOlgr2QH4Beg0hb0xRUd6CO4IKFSohUtQ4+7qbZk6hRC3WN9xUCJf5r",
|
||||
"sdJejUYRVmDs2Ku6mLWgfMFpa1ql7W27zqDu+Kg9umzWVjLPS+E/oTKT9eab45q818bl6eV/AwAA//9q",
|
||||
"XNc2bDQAAA==",
|
||||
"H4sIAAAAAAAC/9xa624bxxV+lcGmQBJ0ScqWW6D8VceOHRm+CKGM/IgFarh7yB1pdmYzM0uaNQTkIfom",
|
||||
"bYD+aH71BZQ3Ks5c9k5RiiU3aRAYK+7MmXP9zmXnQ5TIvJAChNHR9EOkkwxyah8fa81WAtITqi/w7xR0",
|
||||
"olhhmBTRtPWWME0oMfhENWEG/1aQAFtDShZbYjIg30l1AWocxVGhZAHKMLCnJDLPqUjtMzOQ24c/KFhG",
|
||||
"0+izSc3cxHM2eeI2RJdxZLYFRNOIKkW3+Pe5XOBu/7M2iomV/31eKCYVM9vGAiYMrECFFe7Xge2C5sMv",
|
||||
"rqepDTXlXnFQfzO3EiWi+mI3I2XJUnyxlCqnJpq6H+Luwss4UvBDyRSk0fT7sAiV42WpeGuI0NFSQyVN",
|
||||
"ruLaXqfVuXJxDolBBh+vKeN0weGFXMzAGGSn5zkzJlYciHbviVwSSl7IBUFqesBBMskS99im810GgqzY",
|
||||
"GkRMOMuZsX62ppyl+G8JmhiJv2kgnsiYvBF8S0qNPJINMxlxSrOH49mVC/aU33W2FJa05KbP10kGxL90",
|
||||
"fBCdyY3wzJBSgyIb5D0FAypnwp6fMR1UMkbykDKDXDr6/qgl5Rrivh5MBgrpU87lhuDWLk1ClwbXZEDO",
|
||||
"5YJkVJMFgCC6XOTMGEjH5DtZ8pSwvOBbkgIHt41zAu+ZdgSpvtBkKZUjfS4XMaEixViXecE4rmFm/E7U",
|
||||
"PrmQkgMVKNEFbPvKOkpBGLZkoDzdyjFikpfakAWQUrAfSmcuJioRgsV6hqp9/xaaY3kOKaMG+JYoQH8m",
|
||||
"1B6TwpIJhhtidFUrOB4ZW35kadxPBVWGJSWnqrLiDjXochEC/DpcGAilmd9ZOeOtKZz47WumWde3jCqv",
|
||||
"UxD6cNujvC3eHrkQRmUFb1LkC84ugFDyFQeRgiI0TUdSfDkmMzBI7swa5MwFgssYVBBETyUor84wGTV4",
|
||||
"dMlT8bl1hiqWQKQ2lvSwojsgiM7nF90QuGa1nTr4VS5G+Ma5g3PGYHPypFQKhOFbIhFpaKBrvbuBNXpM",
|
||||
"zr55PPvm66fzZ0cvv54fPz755szl0ZQpSIxUW1JQk5E/krN30eQz+9+76IzQokCVpk5sEGWO8i0Zhzmu",
|
||||
"j+IoZSo82p895mdUZ5DO65WnA8Gzy2n6KOc10JC+EbEOYKkmR0+PHZpvrdjoNN4lxuS1JAK0gRQVUyam",
|
||||
"VKDJFxZgdUxSluBRVDHQXxKqgOiyKKQyXdE98zHm3sOHKDSX1ESx9YW9Qg5LF/JRfaarY5gmr6igK1AO",
|
||||
"+ZixoU9zhPKB5MXpAvjtigqvzJsXRENJt5evOuHgXcKx1zhzX2ygtgZS8UumTXAG69279dbXUSg0fp3E",
|
||||
"Jy1E3CFufcSQgKGi7InlXxAFhQKNLBBKtCtffB1kkeg9JKWBfZXujSzeYW7YbNea62ulpEJS3To7hVbt",
|
||||
"GKKlX7jmoDVdDfHaYcfSrNcPcfPCleSU8zfLaPr99XadhWIEd13GPREUUANDdsIXTApiWA7a0LxAFAiC",
|
||||
"ptTACN8MFQtsgNzbt0dPA7i/sFXznoL7prU+BmhV6pdFesfSdKxjOQ06q8+rmD29PHUGegWGptRQa6g0",
|
||||
"tcUO5cct3fck7nSDasGMompLck/MJzs9Jq+ksuFScHjfRPqECswVucRi0+JEibFFzuh4MU7OiJDG6SEU",
|
||||
"hhewxaiC9xRpeRe3jjaNZoViBsgzxVYZYj9WBmPIKePI9XahQPx14ROPVKuwwsVkNLMLyMz8599r4A04",
|
||||
"aTnyrBGnw3pyNdTg3spBQtqiiWFr21FRkaAGXHNVcDD+WThlMSlGS8rciuqhoKW2Dz+UUNoHqpIMO+7q",
|
||||
"0WVFR36EnmGTrSfS+sE+OyolqmjUPDyKow21HcVoKdUI6wc9mFa/hRXTBhSkDgL7IETTVIEedihOtZlb",
|
||||
"pbQ76kbKZMnF7l6cU4NBMoywcmk2VO2A3xvFrhOpDt8qwc2r7ridwPY2kB/VzVe6iCulNrv6oIw4SlxB",
|
||||
"armMulpuaGaHREOYPoOkVMxsd2SaG6eP6/JGKxUMlmd1Y1Y3sZiNn3Gag0hkByryBsjdH2z4F4dXfye/",
|
||||
"/Hj109XPV/+8+umXH6/+dfXz1T+a45bpnw7aid+fMk/yNJpGH/yfl2jBrBQXc83+BtH0EGUyiiZmTsuU",
|
||||
"yQA5GJS2pp9GE2V3TvRyci4X6MAg4MHDw7El2Uwlx6+f45+FjqYPH8XREstYHU2jB6MHB1hO53QFei7V",
|
||||
"fM1SkFgj2F+iOJKlKUrjWgl4b0BoZ5dxYSHHcTB3q9osuUMqphpxoRmaauQFH7ktbsrW9q7ajntybZXX",
|
||||
"bjrDq3phNM7AQK9hrn1pPixt9OrXB4MPZj9lq7gaio3GyPAW+aTKHBXUY+zXmeUmecInnSHwR6be2hJj",
|
||||
"oFes3hE7PxAGsz31lTIGrStO3PjHSkbelQcHD/9MuFxpN1+w42VmPte+3raDsq53NPNHm4c3AkacCT/t",
|
||||
"ESlL8MBNRpFiUnXtmW2vsQyx00FkCA8ekzdrUBsEC00KBWsmS823TpZwaFXyDFWIXA6MQl/KFUGmGkM1",
|
||||
"j9Pt/XG0YZxjtRS6f5TC6sZyAFRxhj3HVJSc+zny7Nbz56G6x9nIpXZFHd9dxP+IxAyJAjP86iMTbCe+",
|
||||
"/Emt3Dh4RCO3nu7Ux4ytxJvbaiLk2vkalPaKvG+xG3XCDml7XF0jtaEGnmRUrKAvuovbeQ0Xtyqoutbq",
|
||||
"ErsRU+kuru6Alz0ctLFYG6qMC1y6oRe2StMcADs5sFVTHOmsNKnc2NklaL9aLpeIBwMI64LF1l0z5NqJ",
|
||||
"t7EMzGmJqb/Xx2pQaHsEXQQyt5gcPY1JQbXeSJWGVy463NcUQk1Yqhphj6hm9WXHrFSzpIapzJgiukQe",
|
||||
"mVhKN24QhiamnntEoS4jJ0Ax+ErF/U49nUyWoWpjctJvL791Q+RnVOUkd3Mk8vj4COtZloDQ0Djn+fHL",
|
||||
"9WGP/mazGa9EiUXcxO/Rk1XBR4fjgzGIcWZy1/cxw1vc+uOiOKqiNnowPhgf4GpZgKAFw4rP/oQZ0mTW",
|
||||
"MhNaMFuAWZ+U2qoCPdMq8yh1g+ScGTdh8J7+lUy3QX0g7B5aFByTFZNicq4daji/3efV7XHKZU+rdsgp",
|
||||
"ffUcNZ0ei0obBbqQqCk86eHBwSflbEM10WWSgF6WnG+J+8QEKWHCZ/A1S0vK3VepceeT3J2w6RqcAf7s",
|
||||
"CxL6FxubZZ5Tta2sSigRsLEDUUztlTv5KWhjbGirBYpVpZ1T6ui0Re5F+Kyi0fkIiLSQTBgrb+VjkypN",
|
||||
"rGDA0Z6DqWa392jV/qB4QHXVonpY3FHgczCE9wbKdtaaAVOdefs1qquPqtR/Xn9nbunvw7lczFl6uVOF",
|
||||
"z8AkmQvV+nw70GQolf/c4iHIEetFVNzQ476m//Qe7XRN0Fn4bpvDSm5fELpw3zut7W7gt26TSD2I5sh5",
|
||||
"ULvLMBPlZ0ajTT0yGgTLMFzyo6X7QcyBYndAUXXbErj/pODZG7MNsCjQvTgJPHxScCwFvC8gMZAS8Gua",
|
||||
"jhHY9wi5CfYMvuR/OB3Y5EyCuFDv1F2P0mwlRnK5vCbvYvG+XPah8FG/hvrtKdIXgRZ7WuXf96eIGrXO",
|
||||
"XlF10az7KDbSrrzco+0nlPuJvPMwe5GGgwv9kMEuhL0QANvPFZCVdFd5LPnxsEnEHouIew1qf8TucK4G",
|
||||
"S58ylvt91e8imG/sg49Lk4EwbtjiRzroDeHyyKb6VnrHDqmApltchfTct/rWmInVBu+7q/FTrMFKoGGy",
|
||||
"6H/tGZZTktj3pG6WL+NdYEZ27/htu9Tt3SPJILkgm3CDKQMF7pbRdocShv1glDRGC4PgNTCGuFcgax40",
|
||||
"oN7XVWp0ct4Az/6/8p7Hc283p4QxOcmYJom94riwN5NogoDBIXWFqRsyeyyph94tX4mJVIhcQSsBX0CN",
|
||||
"uEwot9BGub5rPFtDS5pS91zV+DvaO9JrkkFacjhxs+X7awCbN8YHDGvvijc7311A9Vr6u6bt+3N2QB6u",
|
||||
"11zG0aODw7sbSbQ+ag4wfwwqNOFPQTAHmo8O/jJwMdk5INNESBMynfs849wpJlqG1/bKLrTuETnR7SdJ",
|
||||
"IuTGifrw8NOmlhBFVCCXcmEoE7bsttzFZFEad91vJe3NYyEtzrpou2XEvnHUaUW/oY19oWR9SnsHVwPz",
|
||||
"kUaETD7Yybfv84djpfEd6yatvif48b3+3aeLhiS7YtHXQ9inI4vussevyBYnGQRaGwutCRQhow6GyIn/",
|
||||
"rmYzskeNphs5o9k4MW3aNmaa9H8vaelt/YkTJY+J2RYsoZxvW18kCyVXCrSO/ZUpf/NckSVlvFSwN7eE",
|
||||
"jKJBpK2xDao7UEcUw4ooRKpaBx93Y/NJ1Ci6usb7ChEZ/7egbO9gowgrMHa+Vt0AW1C+4LQ1FtP2Wl9n",
|
||||
"Inh81J6RNos4meel8N9qmcl6g9RxTd5r4/L08r8BAAD//72/8t7VNAAA",
|
||||
}
|
||||
|
||||
// GetSwagger returns the content of the embedded swagger specification file
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user