Start of database interface

Contains a dummy migration because the migrate package will error out if
it cannot find any migration files.

.gitignore

@@ -1,2 +1,3 @@
 *.exe
 /flamenco*-poc
+/*.sqlite

README.md

@@ -19,3 +19,23 @@ You should now have two executables: `flamenco-manager-poc` and `flamenco-worker
 ## Swagger UI
 
 Flamenco Manager has a SwaggerUI interface at http://localhost:8080/api/swagger-ui/
+
+## Flamenco Manager DB migrations
+
+First install the `migrate` tool:
+
+```
+go install -tags sqlite github.com/golang-migrate/migrate/v4/cmd/migrate
+```
+
+To create a migration called `create_users_table`, run:
+
+```
+migrate create -dir internal/manager/persistence/migrations -ext sql -seq create_users_table
+```
+
+Migrations are **automatically run when Flamenco Manager starts**. To run them manually, use:
+
+```
+migrate -database sqlite://flamenco-manager.sqlite -path internal/manager/persistence/migrations up
+```

cmd/flamenco-manager-poc/main.go

@@ -39,6 +39,7 @@ import (
 	"gitlab.com/blender/flamenco-goja-test/internal/appinfo"
 	"gitlab.com/blender/flamenco-goja-test/internal/manager/api_impl"
 	"gitlab.com/blender/flamenco-goja-test/internal/manager/job_compilers"
+	"gitlab.com/blender/flamenco-goja-test/internal/manager/persistence"
 	"gitlab.com/blender/flamenco-goja-test/internal/manager/swagger_ui"
 	"gitlab.com/blender/flamenco-goja-test/pkg/api"
 )
@@ -46,57 +47,52 @@ import (
 func main() {
 	output := zerolog.ConsoleWriter{Out: colorable.NewColorableStdout(), TimeFormat: time.RFC3339}
 	log.Logger = log.Output(output)
-
 	log.Info().Str("version", appinfo.ApplicationVersion).Msgf("starting %v", appinfo.ApplicationName)
 
-	echoOpenAPIPoC()
+	// Open the database.
-}
+	dbCtx, dbCtxCancel := context.WithTimeout(context.Background(), 5*time.Second)
-
+	defer dbCtxCancel()
-// Proof of concept of job compiler in JavaScript.
+	_, err := persistence.OpenDB(dbCtx)
-func gojaPoC() {
-	compiler, err := job_compilers.Load()
 	if err != nil {
-		log.Fatal().Err(err).Msg("error loading job compilers")
+		log.Fatal().Err(err).Msg("error opening database")
 	}
 
-	if err := compiler.Run("simple-blender-render"); err != nil {
+	// TODO: load port number from the configuration.
-		log.Fatal().Err(err).Msg("error running job compiler")
+	// TODO: enable TLS via Let's Encrypt.
-	}
-}
-
-// Proof of concept of a REST API with Echo and OpenAPI.
-func echoOpenAPIPoC() {
 	listen := ":8080"
 	_, port, _ := net.SplitHostPort(listen)
 	log.Info().Str("port", port).Msg("listening")
 
+	// Construct the services.
+	compiler, err := job_compilers.Load()
+	if err != nil {
+		log.Fatal().Err(err).Msg("error loading job compilers")
+	}
+	flamenco := api_impl.NewFlamenco(compiler)
+	e := buildWebService(flamenco)
+
+	// Start the web server.
+	finalErr := e.Start(listen)
+	log.Warn().Err(finalErr).Msg("shutting down")
+}
+
+func buildWebService(flamenco api.ServerInterface) *echo.Echo {
 	e := echo.New()
 	e.HideBanner = true
+
+	// Hook Zerolog onto Echo:
 	e.Use(lecho.Middleware(lecho.Config{
 		Logger: lecho.From(log.Logger),
 	}))
+
+	// Ensure panics when serving a web request won't bring down the server.
 	e.Use(middleware.Recover())
 
-	swagger_ui.RegisterSwaggerUIStaticFiles(e)
+	// Load the API definition and enable validation & authentication checks.
-
 	swagger, err := api.GetSwagger()
 	if err != nil {
 		log.Fatal().Err(err).Msg("unable to get swagger")
 	}
-	e.GET("/api/openapi3.json", func(c echo.Context) error {
-		return c.JSON(http.StatusOK, swagger)
-	})
-
-	e.GET("/api/ping", func(c echo.Context) error {
-		logger := log.Level(zerolog.InfoLevel)
-		logger.Debug().Msg("debug debug")
-		logger.Info().Msg("Info Info")
-
-		return c.JSON(http.StatusOK, echo.Map{
-			"message": "pong",
-		})
-	})
-
 	validator := oapi_middle.OapiRequestValidatorWithOptions(swagger,
 		&oapi_middle.Options{
 			Options: openapi3filter.Options{
@@ -112,13 +108,12 @@ func echoOpenAPIPoC() {
 		})
 	e.Use(validator)
 
-	compiler, err := job_compilers.Load()
+	// Register routes.
-	if err != nil {
-		log.Fatal().Err(err).Msg("error loading job compilers")
-	}
-
-	flamenco := api_impl.NewFlamenco(compiler)
 	api.RegisterHandlers(e, flamenco)
+	swagger_ui.RegisterSwaggerUIStaticFiles(e)
+	e.GET("/api/openapi3.json", func(c echo.Context) error {
+		return c.JSON(http.StatusOK, swagger)
+	})
 
 	// Log available routes
 	routeLogger := log.Level(zerolog.DebugLevel)
@@ -127,8 +122,7 @@ func echoOpenAPIPoC() {
 		routeLogger.Debug().Msgf("%7s %s", route.Method, route.Path)
 	}
 
-	finalErr := e.Start(listen)
+	return e
-	log.Warn().Err(finalErr).Msg("shutting down")
 }
 
 func authenticator(ctx context.Context, authInfo *openapi3filter.AuthenticationInput) error {

go.mod

@@ -7,10 +7,12 @@ require (
 	github.com/dop251/goja v0.0.0-20211217115348-3f9136fa235d
 	github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7
 	github.com/getkin/kin-openapi v0.88.0
+	github.com/golang-migrate/migrate/v4 v4.15.1 // indirect
 	github.com/google/uuid v1.3.0
 	github.com/labstack/echo/v4 v4.6.1
 	github.com/mattn/go-colorable v0.1.12
 	github.com/rs/zerolog v1.26.1
 	github.com/stretchr/testify v1.7.0
 	github.com/ziflex/lecho/v3 v3.1.0
+	modernc.org/sqlite v1.14.4 // indirect
 )

internal/manager/persistence/db.go

@@ -0,0 +1,61 @@
+// Package persistence provides the database interface for Flamenco Manager.
+package persistence
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"github.com/rs/zerolog/log"
+	_ "modernc.org/sqlite"
+)
+
+// TODO : have this configurable from the CLI.
+const dbURI = "flamenco-manager.sqlite"
+
+// DB provides the database interface.
+type DB struct {
+	sqldb *sql.DB
+}
+
+func OpenDB(ctx context.Context) (*DB, error) {
+	log.Info().Str("uri", dbURI).Msg("opening database")
+
+	sqldb, err := sql.Open("sqlite", dbURI)
+	if err != nil {
+		return nil, fmt.Errorf("unable to open database: %w", err)
+	}
+
+	if err := sqldb.PingContext(ctx); err != nil {
+		return nil, fmt.Errorf("error accessing database %s: %w", dbURI, err)
+	}
+
+	db := DB{
+		sqldb: sqldb,
+	}
+	if err := db.migrate(); err != nil {
+		return nil, err
+	}
+
+	return &db, err
+}

internal/manager/persistence/db_migration.go

@@ -0,0 +1,68 @@
+package persistence
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+import (
+	"embed"
+	"fmt"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database/sqlite"
+	"github.com/golang-migrate/migrate/v4/source"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+)
+
+func init() {
+	source.Register("embedfs", &EmbedFS{})
+}
+
+type EmbedFS struct {
+	iofs.PartialDriver
+}
+
+//go:embed migrations/*.sql
+var embedFS embed.FS
+
+func (db *DB) migrate() error {
+	driver, err := sqlite.WithInstance(db.sqldb, &sqlite.Config{})
+	if err != nil {
+		return fmt.Errorf("cannot create migration driver: %w", err)
+	}
+
+	m, err := migrate.NewWithDatabaseInstance("embedfs://", "sqlite", driver)
+	if err != nil {
+		return fmt.Errorf("cannot create migration instance: %w", err)
+	}
+
+	err = m.Up()
+	if err != nil {
+		return fmt.Errorf("cannot migrate database: %w", err)
+	}
+	return nil
+}
+
+func (f *EmbedFS) Open(url string) (source.Driver, error) {
+	nf := &EmbedFS{}
+	if err := nf.Init(embedFS, "migrations"); err != nil {
+		return nil, err
+	}
+	return nf, nil
+}