From 270c54fdb7282b9a3485132afc59b70c06026797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 15 Feb 2022 17:46:37 +0100 Subject: [PATCH] More status change acks & checks to get stable flow between worker states --- internal/manager/api_impl/test_support.go | 16 ++- internal/manager/api_impl/workers.go | 47 ++++++-- internal/manager/api_impl/workers_test.go | 102 ++++++++++++++++ .../persistence/task_scheduler_test.go | 5 - internal/manager/persistence/workers.go | 9 +- internal/worker/state_asleep.go | 38 ++---- internal/worker/state_awake.go | 13 +- internal/worker/statemonitor.go | 56 +++++++++ pkg/api/flamenco-manager.yaml | 2 + pkg/api/openapi_spec.gen.go | 112 +++++++++--------- 10 files changed, 289 insertions(+), 111 deletions(-) create mode 100644 internal/manager/api_impl/workers_test.go create mode 100644 internal/worker/statemonitor.go diff --git a/internal/manager/api_impl/test_support.go b/internal/manager/api_impl/test_support.go index 8e69364f..fa5c4127 100644 --- a/internal/manager/api_impl/test_support.go +++ b/internal/manager/api_impl/test_support.go @@ -3,6 +3,7 @@ package api_impl import ( "bytes" "encoding/json" + "io" "net/http" "net/http/httptest" @@ -53,16 +54,21 @@ func newMockedFlamenco(mockCtrl *gomock.Controller) mockedFlamenco { } func (mf *mockedFlamenco) prepareMockedJSONRequest(worker *persistence.Worker, requestBody interface{}) echo.Context { - - e := echo.New() - bodyBytes, err := json.MarshalIndent(requestBody, "", " ") if err != nil { panic(err) } - req := httptest.NewRequest(http.MethodPost, "/", bytes.NewBuffer(bodyBytes)) - req.Header.Add(echo.HeaderContentType, "application/json") + c := mf.prepareMockedRequest(worker, bytes.NewBuffer(bodyBytes)) + c.Request().Header.Add(echo.HeaderContentType, "application/json") + + return c +} + +func (mf *mockedFlamenco) prepareMockedRequest(worker *persistence.Worker, body io.Reader) echo.Context { + e := echo.New() + + req := httptest.NewRequest(http.MethodPost, "/", body) rec := httptest.NewRecorder() c := e.NewContext(req, rec) requestWorkerStore(c, worker) diff --git a/internal/manager/api_impl/workers.go b/internal/manager/api_impl/workers.go index db18e8c9..87cc649d 100644 --- a/internal/manager/api_impl/workers.go +++ b/internal/manager/api_impl/workers.go @@ -21,6 +21,7 @@ package api_impl * ***** END GPL LICENSE BLOCK ***** */ import ( + "fmt" "net/http" "strings" @@ -143,6 +144,10 @@ func (f *Flamenco) SignOff(e echo.Context) error { logger.Info().Msg("worker signing off") w := requestWorkerOrPanic(e) w.Status = api.WorkerStatusOffline + if w.StatusRequested == api.WorkerStatusShutdown { + w.StatusRequested = "" + } + // TODO: check whether we should pass the request context here, or a generic // background context, as this should be stored even when the HTTP connection // is aborted. @@ -183,10 +188,24 @@ func (f *Flamenco) WorkerStateChanged(e echo.Context) error { return sendAPIError(e, http.StatusBadRequest, "invalid format") } - logger.Info().Str("newStatus", string(req.Status)).Msg("worker changed status") - w := requestWorkerOrPanic(e) + logger = logger.With(). + Str("currentStatus", string(w.Status)). + Str("newStatus", string(req.Status)). + Logger() + w.Status = req.Status + if w.StatusRequested != "" && req.Status != w.StatusRequested { + logger.Warn(). + Str("workersRequestedStatus", string(w.StatusRequested)). + Msg("worker changed to status that was not requested") + } else { + logger.Info().Msg("worker changed status") + // Either there was no status change request (and this is a no-op) or the + // status change was actually acknowledging the request. + w.StatusRequested = "" + } + err = f.persist.SaveWorkerStatus(e.Request().Context(), w) if err != nil { logger.Warn().Err(err). @@ -200,13 +219,27 @@ func (f *Flamenco) WorkerStateChanged(e echo.Context) error { func (f *Flamenco) ScheduleTask(e echo.Context) error { logger := requestLogger(e) + worker := requestWorkerOrPanic(e) logger.Info().Msg("worker requesting task") - // Figure out which worker is requesting a task: - worker := requestWorker(e) - if worker == nil { - logger.Warn().Msg("task requested by non-worker") - return sendAPIError(e, http.StatusBadRequest, "not authenticated as Worker") + // Check that this worker is actually allowed to do work. + requiredStatusToGetTask := api.WorkerStatusAwake + if worker.Status != api.WorkerStatusAwake { + logger.Warn(). + Str("workerStatus", string(worker.Status)). + Str("requiredStatus", string(requiredStatusToGetTask)). + Msg("worker asking for task but is in wrong state") + return sendAPIError(e, http.StatusConflict, + fmt.Sprintf("worker is in state %q, requires state %q to execute tasks", worker.Status, requiredStatusToGetTask)) + } + if worker.StatusRequested != "" { + logger.Warn(). + Str("workerStatus", string(worker.Status)). + Str("requestedStatus", string(worker.StatusRequested)). + Msg("worker asking for task but needs state change first") + return e.JSON(http.StatusLocked, api.WorkerStateChange{ + StatusRequested: worker.StatusRequested, + }) } // Get a task to execute: diff --git a/internal/manager/api_impl/workers_test.go b/internal/manager/api_impl/workers_test.go new file mode 100644 index 00000000..2616f2de --- /dev/null +++ b/internal/manager/api_impl/workers_test.go @@ -0,0 +1,102 @@ +package api_impl + +/* ***** BEGIN GPL LICENSE BLOCK ***** + * + * Original Code Copyright (C) 2022 Blender Foundation. + * + * This file is part of Flamenco. + * + * Flamenco is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * Flamenco. If not, see . + * + * ***** END GPL LICENSE BLOCK ***** */ + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "gitlab.com/blender/flamenco-ng-poc/internal/manager/persistence" + "gitlab.com/blender/flamenco-ng-poc/pkg/api" +) + +func TestTaskScheduleHappy(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mf := newMockedFlamenco(mockCtrl) + worker := testWorker() + + // Expect a call into the persistence layer, which should return a scheduled task. + job := persistence.Job{ + UUID: "583a7d59-887a-4c6c-b3e4-a753018f71b0", + } + task := persistence.Task{ + UUID: "4107c7aa-e86d-4244-858b-6c4fce2af503", + Job: &job, + } + mf.persistence.EXPECT().ScheduleTask(&worker).Return(&task, nil) + + echoCtx := mf.prepareMockedRequest(&worker, nil) + err := mf.flamenco.ScheduleTask(echoCtx) + assert.NoError(t, err) + + resp := echoCtx.Response().Writer.(*httptest.ResponseRecorder) + assert.Equal(t, http.StatusOK, resp.Code) + // TODO: check that the returned JSON actually matches what we expect. +} + +func TestTaskScheduleNonActiveStatus(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mf := newMockedFlamenco(mockCtrl) + worker := testWorker() + worker.Status = api.WorkerStatusAsleep + + // Explicitly NO expected calls to the persistence layer. Since the worker is + // not in a state that allows task execution, there should be no DB queries. + + echoCtx := mf.prepareMockedRequest(&worker, nil) + err := mf.flamenco.ScheduleTask(echoCtx) + assert.NoError(t, err) + + resp := echoCtx.Response().Writer.(*httptest.ResponseRecorder) + assert.Equal(t, http.StatusConflict, resp.Code) +} + +func TestTaskScheduleOtherStatusRequested(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mf := newMockedFlamenco(mockCtrl) + worker := testWorker() + worker.StatusRequested = api.WorkerStatusAsleep + + // Explicitly NO expected calls to the persistence layer. Since the worker is + // not in a state that allows task execution, there should be no DB queries. + + echoCtx := mf.prepareMockedRequest(&worker, nil) + err := mf.flamenco.ScheduleTask(echoCtx) + assert.NoError(t, err) + + resp := echoCtx.Response().Writer.(*httptest.ResponseRecorder) + assert.Equal(t, http.StatusLocked, resp.Code) + + responseBody := api.WorkerStateChange{} + err = json.Unmarshal(resp.Body.Bytes(), &responseBody) + assert.NoError(t, err) + assert.Equal(t, worker.StatusRequested, responseBody.StatusRequested) +} diff --git a/internal/manager/persistence/task_scheduler_test.go b/internal/manager/persistence/task_scheduler_test.go index 2ee6af64..1a5db628 100644 --- a/internal/manager/persistence/task_scheduler_test.go +++ b/internal/manager/persistence/task_scheduler_test.go @@ -158,11 +158,6 @@ func TestTwoJobsThreeTasks(t *testing.T) { assert.Equal(t, att2_3.Name, task.Name, "the 3rd task of the 2nd job should have been chosen") } -// To test: worker with non-active state. -// Unlike Flamenco v2, this Manager shouldn't change a worker's status -// simply because it requests a task. New tasks for non-awake workers -// should be rejected. - // To test: blacklists // To test: variable replacement diff --git a/internal/manager/persistence/workers.go b/internal/manager/persistence/workers.go index 4c1f3ef0..37ecd8b4 100644 --- a/internal/manager/persistence/workers.go +++ b/internal/manager/persistence/workers.go @@ -67,7 +67,14 @@ func (db *DB) FetchWorker(ctx context.Context, uuid string) (*Worker, error) { } func (db *DB) SaveWorkerStatus(ctx context.Context, w *Worker) error { - if err := db.gormDB.Model(w).Select("status").Updates(Worker{Status: w.Status}).Error; err != nil { + err := db.gormDB. + Model(w). + Select("status", "status_requested"). + Updates(Worker{ + Status: w.Status, + StatusRequested: w.StatusRequested, + }).Error + if err != nil { return fmt.Errorf("error saving worker: %w", err) } return nil diff --git a/internal/worker/state_asleep.go b/internal/worker/state_asleep.go index e1841bb9..3bfe1ee5 100644 --- a/internal/worker/state_asleep.go +++ b/internal/worker/state_asleep.go @@ -22,7 +22,6 @@ package worker import ( "context" - "net/http" "time" "github.com/rs/zerolog/log" @@ -37,7 +36,7 @@ func (w *Worker) gotoStateAsleep(ctx context.Context) { w.state = api.WorkerStatusAsleep w.doneWg.Add(2) - go w.ackStateChange(ctx, w.state) + w.ackStateChange(ctx, w.state) go w.runStateAsleep(ctx) } @@ -49,37 +48,18 @@ func (w *Worker) runStateAsleep(ctx context.Context) { for { select { case <-ctx.Done(): - logger.Debug().Msg("state fetching interrupted by context cancellation") + logger.Debug().Msg("asleep state interrupted by context cancellation") return case <-w.doneChan: - logger.Debug().Msg("state fetching interrupted by shutdown") + logger.Debug().Msg("asleep state interrupted by shutdown") return case <-time.After(durationSleepCheck): - } - if !w.isState(api.WorkerStatusAsleep) { - logger.Debug().Str("newStatus", string(w.state)).Msg("state fetching interrupted by state change") - return - } - - resp, err := w.client.WorkerStateWithResponse(ctx) - if err != nil { - log.Warn().Err(err).Msg("error checking upstream state changes") - continue - } - switch { - case resp.JSON200 != nil: - log.Info(). - Str("requestedStatus", string(resp.JSON200.StatusRequested)). - Msg("Manager requests status change") - w.changeState(ctx, resp.JSON200.StatusRequested) - return - case resp.StatusCode() == http.StatusNoContent: - log.Debug().Msg("we can keep sleeping") - default: - log.Warn(). - Int("code", resp.StatusCode()). - Str("error", string(resp.Body)). - Msg("unable to obtain requested state for unknown reason") + newStatus := w.queryManagerForStateChange(ctx) + if newStatus != nil { + logger.Debug().Str("newStatus", string(*newStatus)).Msg("asleep state interrupted by state change") + w.changeState(ctx, *newStatus) + return + } } } } diff --git a/internal/worker/state_awake.go b/internal/worker/state_awake.go index 8dee12db..5f181102 100644 --- a/internal/worker/state_awake.go +++ b/internal/worker/state_awake.go @@ -42,7 +42,7 @@ func (w *Worker) gotoStateAwake(ctx context.Context) { w.state = api.WorkerStatusAwake w.doneWg.Add(2) - go w.ackStateChange(ctx, w.state) + w.ackStateChange(ctx, w.state) go w.runStateAwake(ctx) } @@ -62,6 +62,9 @@ func (w *Worker) runStateAwake(ctx context.Context) { if err != nil { log.Warn().Err(err).Interface("task", *task).Msg("error executing task") } + + // Do some rate limiting. This is mostly useful while developing. + time.Sleep(2 * time.Second) } } @@ -84,10 +87,6 @@ func (w *Worker) fetchTask(ctx context.Context) *api.AssignedTask { return nil case <-time.After(wait): } - if !w.isState(api.WorkerStatusAwake) { - logger.Debug().Msg("task fetching interrupted by state change") - return nil - } resp, err := w.client.ScheduleTaskWithResponse(ctx) if err != nil { @@ -112,18 +111,16 @@ func (w *Worker) fetchTask(ctx context.Context) *api.AssignedTask { Str("error", string(resp.JSON403.Message)). Msg("access denied") wait = durationFetchFailed - continue case resp.StatusCode() == http.StatusNoContent: log.Info().Msg("no task available") wait = durationNoTask - continue default: log.Warn(). Int("code", resp.StatusCode()). Str("error", string(resp.Body)). Msg("unable to obtain task for unknown reason") wait = durationFetchFailed - continue } + } } diff --git a/internal/worker/statemonitor.go b/internal/worker/statemonitor.go new file mode 100644 index 00000000..a9cc5614 --- /dev/null +++ b/internal/worker/statemonitor.go @@ -0,0 +1,56 @@ +package worker + +/* ***** BEGIN GPL LICENSE BLOCK ***** + * + * Original Code Copyright (C) 2022 Blender Foundation. + * + * This file is part of Flamenco. + * + * Flamenco is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * Flamenco. If not, see . + * + * ***** END GPL LICENSE BLOCK ***** */ + +import ( + "context" + "net/http" + + "github.com/rs/zerolog/log" + "gitlab.com/blender/flamenco-ng-poc/pkg/api" +) + +// queryManagerForStateChange asks the Manager whether we should go to another state or not. +// Any error communicating with the Manager is logged but otherwise ignored. +// Returns nil when no state change is requested. +func (w *Worker) queryManagerForStateChange(ctx context.Context) *api.WorkerStatus { + resp, err := w.client.WorkerStateWithResponse(ctx) + if err != nil { + log.Warn().Err(err).Msg("error checking upstream state changes") + return nil + } + switch { + case resp.JSON200 != nil: + log.Info(). + Str("requestedStatus", string(resp.JSON200.StatusRequested)). + Msg("Manager requests status change") + return &resp.JSON200.StatusRequested + case resp.StatusCode() == http.StatusNoContent: + log.Debug().Msg("we can stay in the current state") + default: + log.Warn(). + Int("code", resp.StatusCode()). + Str("error", string(resp.Body)). + Msg("unable to obtain requested state for unknown reason") + } + + return nil +} diff --git a/pkg/api/flamenco-manager.yaml b/pkg/api/flamenco-manager.yaml index d09c4a25..e52bb4eb 100644 --- a/pkg/api/flamenco-manager.yaml +++ b/pkg/api/flamenco-manager.yaml @@ -150,6 +150,8 @@ paths: content: application/json: schema: {$ref: "#/components/schemas/SecurityError"} + "409": + description: Worker is not in the active state, so is not allowed to execute tasks right now. "423": description: Worker cannot obtain new tasks, but must go to another state. content: diff --git a/pkg/api/openapi_spec.gen.go b/pkg/api/openapi_spec.gen.go index 0d104f1e..e9487fab 100644 --- a/pkg/api/openapi_spec.gen.go +++ b/pkg/api/openapi_spec.gen.go @@ -18,62 +18,62 @@ import ( // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/9xb624bxxV+lcGmQBJ0ScqWW6D8VceOHRm+CKGM/IgFarh7yB1pdmYzM0uaNQTkIfom", - "bYD+aH71BZQ3Ks5c9k5RiiU3aRAEK+7MmXP9zmUnH6JE5oUUIIyOph8inWSQU/v4WGu2EpCeUH2Bf6eg", - "E8UKw6SIpq23hGlCicEnqgkz+LeCBNgaUrLYEpMB+U6qC1DjKI4KJQtQhoE9JZF5TkVqn5mB3D78QcEy", - "mkafTWrmJp6zyRO3IbqMI7MtIJpGVCm6xb/P5QJ3+5+1UUys/O/zQjGpmNk2FjBhYAUqrHC/DmwXNB9+", - "cT1Nbagp94qD+pu5lSgR1Re7GSlLluKLpVQ5NdHU/RB3F17GkYIfSqYgjabfh0WoHC9LxVtDhI6WGipp", - "chXX9jqtzpWLc0gMMvh4TRmnCw4v5GIGxiA7Pc+ZMbHiQLR7T+SSUPJCLghS0wMOkkmWuMc2ne8yEGTF", - "1iBiwlnOjPWzNeUsxf+WoImR+JsG4omMyRvBt6TUyCPZMJMRpzR7OJ5duWBP+V1nS2FJS276fJ1kQPxL", - "xwfRmdwIzwwpNSiyQd5TMKByJuz5GdNBJWMkDykzyKWj749aUq4h7uvBZKCQPuVcbghu7dIkdGlwTQbk", - "XC5IRjVZAAiiy0XOjIF0TL6TJU8Jywu+JSlwcNs4J/CeaUeQ6gtNllI50udyERMqUox1mReM4xpmxu9E", - "7ZMLKTlQgRJdwLavrKMUhGFLBsrTrRwjJnmpDVkAKQX7oXTmYqISIVisZ6ja92+hOZbnkDJqgG+JAvRn", - "Qu0xKSyZYLghRle1guORseVHlsb9VFBlWFJyqior7lCDLhchwK/DhYFQmvmdlTPemsKJ375mmnV9y6jy", - "OgWhD7c9ytvi7ZELYVRW8CZFvuDsAgglX3EQKShC03QkxZdjMgOD5M6sQc5cILiMQQVB9FSC8uoMk1GD", - "R5c8FZ9bZ6hiCURqY0kPK7oDguh8ftENgWtW26mDX+VihG+cOzhnDDYnT0qlQBi+JRKRhga61rsbWKPH", - "5Oybx7Nvvn46f3b08uv58eOTb85cHk2ZgsRItSUFNRn5Izl7F00+s/+8i84ILQpUaerEBlHmKN+ScZjj", - "+iiOUqbCo/3ZY35GdQbpvF55OhA8u5ymj3JeAw3pGxHrAJZqcvT02KH51oqNTuNdYkxeSyJAG0hRMWVi", - "SgWafGEBVsckZQkeRRUD/SWhCogui0Iq0xXdMx9j7j18iEJzSU0UW1/YK+SwdCEf1We6OoZp8ooKugLl", - "kI8ZG/o0RygfSF6cLoDfrqjwyrx5QTSUdHv5qhMO3iUce40z98UGamsgFb9k2gRnsN69W299HYVC49dJ", - "fNJCxB3i1kcMCRgqyp5Y/gVRUCjQyAKhRLvyxddBFoneQ1Ia2Ffp3sjiHeaGzXatub5WSiok1a2zU2jV", - "jiFa+oVrDlrT1RCvHXYszXr9EDcvXElOOX+zjKbfX2/XWShGcNdl3BNBATUwZCd8waQghuWgDc0LRIEg", - "aEoNjPDNULHABsi9fXv0NID7C1s17ym4b1rrY4BWpX5ZpHcsTcc6ltOgs/q8itnTy1NnoFdgaEoNtYZK", - "U1vsUH7c0n1P4k43qBbMKKq2JPfEfLLTY/JKKhsuBYf3TaRPqMBckUssNi1OlBhb5IyOF+PkjAhpnB5C", - "YXgBW4wqeE+Rlndx62jTaFYoZoA8U2yVIfZjZTCGnDKOXG8XCsRfFz7xSLUKK1xMRjO7gMzMf/69Bt6A", - "k5YjzxpxOqwnV0MN7q0cJKQtmhi2th0VFQlqwDVXBQfjn4VTFpNitKTMrageClpq+/BDCaV9oCrJsOOu", - "Hl1WdORH6Bk22XoirR/ss6NSoopGzcOjONpQ21GMllKNsH7Qg2n1W1gxbUBB6iCwD0I0TRXoYYfiVJu5", - "VUq7o26kTJZc7O7FOTUYJMMIK5dmQ9UO+L1R7DqR6vCtEty86o7bCWxvA/lR3Xyli7hSarOrD8qIo8QV", - "pJbLqKvlhmZ2SDSE6TNISsXMdkemuXH6uC5vtFLBYHlWN2Z1E4vZ+BmnOYhEdqAib4Dc/cGGf3F49Xfy", - "y49XP139fPXPq59++fHqX1c/X/2jOW6Z/umgnfj9KfMkT6Np9MH/eYkWzEpxMdfsbxBND1Emo2hi5rRM", - "mQyQg0Fpa/ppNFF250QvJ+dygQ4MAh48PBxbks1Ucvz6Of5Z6Gj68FEcLbGM1dE0ejB6cIDldE5XoOdS", - "zdcsBYk1gv0liiNZmqI0rpWA9waEdnYZFxZyHAdzt6rNkjukYqoRF5qhqUZe8JHb4qZsbe+q7bgn11Z5", - "7aYzvKoXRuMMDPQa5tqX5sPSRq9+fTD4YPZTtoqrodhojAxvkU+qzFFBPcZ+nVlukid80hkCf2TqrS0x", - "BnrF6h2x8wNhMNtTXylj0LrixI1/rGTkXXlw8PDPhMuVdvMFO15m5nPt6207KOt6RzN/tHl4I2DEmfDT", - "HpGyBA/cZBQpJlXXntn2GssQOx1EhvDgMXmzBrVBsNCkULBmstR862QJh1Ylz1CFyOXAKPSlXBFkqjFU", - "8zjd3h9HG8Y5Vkuh+0cprG4sB0AVZ9hzTEXJuZ8jz249fx6qe5yNXGpX1PHdRfyPSMyQKDDDrz4ywXbi", - "y5/Uyo2DRzRy6+lOfczYSry5rSZCrp2vQWmvyPsWu1En7JC2x9U1Uhtq4ElGxQr6oru4nddwcauCqmut", - "LrEbMZXu4uoOeNnDQRuLtaHKuMClG3phqzTNAbCTA1s1xZHOSpPKjZ1dgvar5XKJeDCAsC5YbN01Q66d", - "eBvLwJyWmPp7fawGhbZH0EUgc4vJ0dOYFFTrjVRpeOWiw31NIdSEpaoR9ohqVl92zEo1S2qYyowpokvk", - "kYmldOMGYWhi6rlHFOoycgIUg69U3O/U08lkGao2Jif99vJbN0R+RlVOcjdHIo+Pj7CeZQkIDY1znh+/", - "XB/26G82m/FKlFjETfwePVkVfHQ4PhiDGGcmd30fM7zFrT8uiqMqaqMH44PxAa6WBQhaMKz47E+YIU1m", - "LTOhBbMFmPVJqa0q0DOtMo9SN0jOmXETBu/pX8l0G9QHwu6hRcExWTEpJufaoYbz231e3R6nXPa0aoec", - "0lfPUdPpsai0UaALiZrCkx4eHHxSzjZUE10mCehlyfmWuE9MkBImfAZfs7Sk3H2VGnc+yd0Jm67BGeDP", - "viChf7GxWeY5VdvKqoQSARs7EMXUXrmTn4I2xoa2WqBYVdo5pY5OW+RehM8qGp2PgEgLyYSx8lY+NqnS", - "xAoGHO05mGp2e49W7Q+KB1RXLaqHxR0FPgdDeG+gbGetGTDVmbdfo7r6qEr95/V35pb+PpzLxZyllztV", - "+AxMkrlQrc+3A02GUvnPLR6CHLFeRMUNPe5r+k/v0U7XBJ2F77Y5rOT2BaEL973T2u4Gfus2idSDaI6c", - "B7W7DDNRfmY02tQjo0GwDMMlP1q6H8QcKHYHFFW3LYH7TwqevTHbAIsC3YuTwMMnBcdSwPsCEgMpAb+m", - "6RiBfY+Qm2DP4Ev+h9OBTc4kiAv1Tt31KM1WYiSXy2vyLhbvy2UfCh/1a6jfniJ9EWixp1X+fX+KqFHr", - "7BVVF826j2Ij7crLPdp+QrmfyDsPsxdpOLjQDxnsQtgLAbD9XAFZSXeVx5IfD5tE7LGIuNeg9kfsDudq", - "sPQpY7nfV/0ugvnGPvi4NBkI44YtfqSD3hAuj2yqb6V37JAKaLrFVUjPfatvjZlYbfC+uxo/xRqsBBom", - "i/7XnmE5JYl9T+pm+TLeBWZk947ftkvd3j2SDJILsgk3mDJQ4G4ZbXcoYdgPRkljtDAIXgNjiHsFsuZB", - "A+p9XaVGJ+cN8Oz/K+95PPd2c0oYk5OMaZLYK44LezOJJggYHFJXmLohs8eSeujd8pWYSIXIFbQS8AXU", - "iMuEcgttlOu7xrM1tKQpdc9Vjb+jvSO9JhmkJYcTN1u+vwaweWN8wLD2rniz890FVK+lv2vavj9nB+Th", - "es1lHD06OLy7kUTro+YA88egQhP+FARzoPno4eGnRfzg3FQIaYhcGMqErYatvmKyKI27hbeS9kKwkBb+", - "XBDcMpDeOOq0ot+w3T4Pt6bW3u/UwNii4biTD3Yg7dvvYRdufF66SQfuCX58C373KN6QZFeI+DIF22dk", - "0d3B+BUgfpJBoLWxiJdAERLdo4O/DG8w4X/o8MHcdCNntJhof5W9po3e2KL/e8kWb+svjyh5TMy2YAnl", - "fNv6UFgouVKgdexvMvkL4YosKeOlgr2QH4Beg0hb0xRUd6CO4IKFSohUtQ4+7qbZk6hRC3WN9xUCJf5r", - "sdJejUYRVmDs2Ku6mLWgfMFpa1ql7W27zqDu+Kg9umzWVjLPS+E/oTKT9eab45q818bl6eV/AwAA//9q", - "XNc2bDQAAA==", + "H4sIAAAAAAAC/9xa624bxxV+lcGmQBJ0ScqWW6D8VceOHRm+CKGM/IgFarh7yB1pdmYzM0uaNQTkIfom", + "bYD+aH71BZQ3Ks5c9k5RiiU3aRAYK+7MmXP9zmXnQ5TIvJAChNHR9EOkkwxyah8fa81WAtITqi/w7xR0", + "olhhmBTRtPWWME0oMfhENWEG/1aQAFtDShZbYjIg30l1AWocxVGhZAHKMLCnJDLPqUjtMzOQ24c/KFhG", + "0+izSc3cxHM2eeI2RJdxZLYFRNOIKkW3+Pe5XOBu/7M2iomV/31eKCYVM9vGAiYMrECFFe7Xge2C5sMv", + "rqepDTXlXnFQfzO3EiWi+mI3I2XJUnyxlCqnJpq6H+Luwss4UvBDyRSk0fT7sAiV42WpeGuI0NFSQyVN", + "ruLaXqfVuXJxDolBBh+vKeN0weGFXMzAGGSn5zkzJlYciHbviVwSSl7IBUFqesBBMskS99im810GgqzY", + "GkRMOMuZsX62ppyl+G8JmhiJv2kgnsiYvBF8S0qNPJINMxlxSrOH49mVC/aU33W2FJa05KbP10kGxL90", + "fBCdyY3wzJBSgyIb5D0FAypnwp6fMR1UMkbykDKDXDr6/qgl5Rrivh5MBgrpU87lhuDWLk1ClwbXZEDO", + "5YJkVJMFgCC6XOTMGEjH5DtZ8pSwvOBbkgIHt41zAu+ZdgSpvtBkKZUjfS4XMaEixViXecE4rmFm/E7U", + "PrmQkgMVKNEFbPvKOkpBGLZkoDzdyjFikpfakAWQUrAfSmcuJioRgsV6hqp9/xaaY3kOKaMG+JYoQH8m", + "1B6TwpIJhhtidFUrOB4ZW35kadxPBVWGJSWnqrLiDjXochEC/DpcGAilmd9ZOeOtKZz47WumWde3jCqv", + "UxD6cNujvC3eHrkQRmUFb1LkC84ugFDyFQeRgiI0TUdSfDkmMzBI7swa5MwFgssYVBBETyUor84wGTV4", + "dMlT8bl1hiqWQKQ2lvSwojsgiM7nF90QuGa1nTr4VS5G+Ma5g3PGYHPypFQKhOFbIhFpaKBrvbuBNXpM", + "zr55PPvm66fzZ0cvv54fPz755szl0ZQpSIxUW1JQk5E/krN30eQz+9+76IzQokCVpk5sEGWO8i0Zhzmu", + "j+IoZSo82p895mdUZ5DO65WnA8Gzy2n6KOc10JC+EbEOYKkmR0+PHZpvrdjoNN4lxuS1JAK0gRQVUyam", + "VKDJFxZgdUxSluBRVDHQXxKqgOiyKKQyXdE98zHm3sOHKDSX1ESx9YW9Qg5LF/JRfaarY5gmr6igK1AO", + "+ZixoU9zhPKB5MXpAvjtigqvzJsXRENJt5evOuHgXcKx1zhzX2ygtgZS8UumTXAG69279dbXUSg0fp3E", + "Jy1E3CFufcSQgKGi7InlXxAFhQKNLBBKtCtffB1kkeg9JKWBfZXujSzeYW7YbNea62ulpEJS3To7hVbt", + "GKKlX7jmoDVdDfHaYcfSrNcPcfPCleSU8zfLaPr99XadhWIEd13GPREUUANDdsIXTApiWA7a0LxAFAiC", + "ptTACN8MFQtsgNzbt0dPA7i/sFXznoL7prU+BmhV6pdFesfSdKxjOQ06q8+rmD29PHUGegWGptRQa6g0", + "tcUO5cct3fck7nSDasGMompLck/MJzs9Jq+ksuFScHjfRPqECswVucRi0+JEibFFzuh4MU7OiJDG6SEU", + "hhewxaiC9xRpeRe3jjaNZoViBsgzxVYZYj9WBmPIKePI9XahQPx14ROPVKuwwsVkNLMLyMz8599r4A04", + "aTnyrBGnw3pyNdTg3spBQtqiiWFr21FRkaAGXHNVcDD+WThlMSlGS8rciuqhoKW2Dz+UUNoHqpIMO+7q", + "0WVFR36EnmGTrSfS+sE+OyolqmjUPDyKow21HcVoKdUI6wc9mFa/hRXTBhSkDgL7IETTVIEedihOtZlb", + "pbQ76kbKZMnF7l6cU4NBMoywcmk2VO2A3xvFrhOpDt8qwc2r7ridwPY2kB/VzVe6iCulNrv6oIw4SlxB", + "armMulpuaGaHREOYPoOkVMxsd2SaG6eP6/JGKxUMlmd1Y1Y3sZiNn3Gag0hkByryBsjdH2z4F4dXfye/", + "/Hj109XPV/+8+umXH6/+dfXz1T+a45bpnw7aid+fMk/yNJpGH/yfl2jBrBQXc83+BtH0EGUyiiZmTsuU", + "yQA5GJS2pp9GE2V3TvRyci4X6MAg4MHDw7El2Uwlx6+f45+FjqYPH8XREstYHU2jB6MHB1hO53QFei7V", + "fM1SkFgj2F+iOJKlKUrjWgl4b0BoZ5dxYSHHcTB3q9osuUMqphpxoRmaauQFH7ktbsrW9q7ajntybZXX", + "bjrDq3phNM7AQK9hrn1pPixt9OrXB4MPZj9lq7gaio3GyPAW+aTKHBXUY+zXmeUmecInnSHwR6be2hJj", + "oFes3hE7PxAGsz31lTIGrStO3PjHSkbelQcHD/9MuFxpN1+w42VmPte+3raDsq53NPNHm4c3AkacCT/t", + "ESlL8MBNRpFiUnXtmW2vsQyx00FkCA8ekzdrUBsEC00KBWsmS823TpZwaFXyDFWIXA6MQl/KFUGmGkM1", + "j9Pt/XG0YZxjtRS6f5TC6sZyAFRxhj3HVJSc+zny7Nbz56G6x9nIpXZFHd9dxP+IxAyJAjP86iMTbCe+", + "/Emt3Dh4RCO3nu7Ux4ytxJvbaiLk2vkalPaKvG+xG3XCDml7XF0jtaEGnmRUrKAvuovbeQ0Xtyqoutbq", + "ErsRU+kuru6Alz0ctLFYG6qMC1y6oRe2StMcADs5sFVTHOmsNKnc2NklaL9aLpeIBwMI64LF1l0z5NqJ", + "t7EMzGmJqb/Xx2pQaHsEXQQyt5gcPY1JQbXeSJWGVy463NcUQk1Yqhphj6hm9WXHrFSzpIapzJgiukQe", + "mVhKN24QhiamnntEoS4jJ0Ax+ErF/U49nUyWoWpjctJvL791Q+RnVOUkd3Mk8vj4COtZloDQ0Djn+fHL", + "9WGP/mazGa9EiUXcxO/Rk1XBR4fjgzGIcWZy1/cxw1vc+uOiOKqiNnowPhgf4GpZgKAFw4rP/oQZ0mTW", + "MhNaMFuAWZ+U2qoCPdMq8yh1g+ScGTdh8J7+lUy3QX0g7B5aFByTFZNicq4daji/3efV7XHKZU+rdsgp", + "ffUcNZ0ei0obBbqQqCk86eHBwSflbEM10WWSgF6WnG+J+8QEKWHCZ/A1S0vK3VepceeT3J2w6RqcAf7s", + "CxL6FxubZZ5Tta2sSigRsLEDUUztlTv5KWhjbGirBYpVpZ1T6ui0Re5F+Kyi0fkIiLSQTBgrb+VjkypN", + "rGDA0Z6DqWa392jV/qB4QHXVonpY3FHgczCE9wbKdtaaAVOdefs1qquPqtR/Xn9nbunvw7lczFl6uVOF", + "z8AkmQvV+nw70GQolf/c4iHIEetFVNzQ476m//Qe7XRN0Fn4bpvDSm5fELpw3zut7W7gt26TSD2I5sh5", + "ULvLMBPlZ0ajTT0yGgTLMFzyo6X7QcyBYndAUXXbErj/pODZG7MNsCjQvTgJPHxScCwFvC8gMZAS8Gua", + "jhHY9wi5CfYMvuR/OB3Y5EyCuFDv1F2P0mwlRnK5vCbvYvG+XPah8FG/hvrtKdIXgRZ7WuXf96eIGrXO", + "XlF10az7KDbSrrzco+0nlPuJvPMwe5GGgwv9kMEuhL0QANvPFZCVdFd5LPnxsEnEHouIew1qf8TucK4G", + "S58ylvt91e8imG/sg49Lk4EwbtjiRzroDeHyyKb6VnrHDqmApltchfTct/rWmInVBu+7q/FTrMFKoGGy", + "6H/tGZZTktj3pG6WL+NdYEZ27/htu9Tt3SPJILkgm3CDKQMF7pbRdocShv1glDRGC4PgNTCGuFcgax40", + "oN7XVWp0ct4Az/6/8p7Hc283p4QxOcmYJom94riwN5NogoDBIXWFqRsyeyyph94tX4mJVIhcQSsBX0CN", + "uEwot9BGub5rPFtDS5pS91zV+DvaO9JrkkFacjhxs+X7awCbN8YHDGvvijc7311A9Vr6u6bt+3N2QB6u", + "11zG0aODw7sbSbQ+ag4wfwwqNOFPQTAHmo8O/jJwMdk5INNESBMynfs849wpJlqG1/bKLrTuETnR7SdJ", + "IuTGifrw8NOmlhBFVCCXcmEoE7bsttzFZFEad91vJe3NYyEtzrpou2XEvnHUaUW/oY19oWR9SnsHVwPz", + "kUaETD7Yybfv84djpfEd6yatvif48b3+3aeLhiS7YtHXQ9inI4vussevyBYnGQRaGwutCRQhow6GyIn/", + "rmYzskeNphs5o9k4MW3aNmaa9H8vaelt/YkTJY+J2RYsoZxvW18kCyVXCrSO/ZUpf/NckSVlvFSwN7eE", + "jKJBpK2xDao7UEcUw4ooRKpaBx93Y/NJ1Ci6usb7ChEZ/7egbO9gowgrMHa+Vt0AW1C+4LQ1FtP2Wl9n", + "Inh81J6RNos4meel8N9qmcl6g9RxTd5r4/L08r8BAAD//72/8t7VNAAA", } // GetSwagger returns the content of the embedded swagger specification file