From 5f37bcb62930a305a635e5d206809905f1bf04f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Fri, 4 Oct 2024 21:59:44 +0200 Subject: [PATCH] Build with Magefile (#104341) Convert most of the code in `Makefile` to [Magefile](https://magefile.org/): This makes it possible to build Flamenco without `make` (and the POSIX environment/commands it expect) by running: ```bash $ go run mage.go webappInstallDeps # Only on the first build. $ go run mage.go build ``` More efficient builds are possible with other commands, and some release-related commands still require `make`. At least the barrier to entry should be considerably lower (compared to having to install Make + Cygwin/MSYS2 on Windows). Fix: #102633 This does not port the building of release packages, so it doesn't address #102671. ### Main Targets | Target | Description | |----------|---------------------------------------------------------------------------------| | build | Build Flamenco Manager and Flamenco Worker, including the webapp and the add-on | | check | Run unit tests, check for vulnerabilities, and run the linter | | clean | Remove executables and other build output | | generate | Generate code (OpenAPI and test mocks) | ### All Targets Get these via `go run mage.go -l`: ``` Targets: build Flamenco Manager and Flamenco Worker, including the webapp and the add-on check Run unit tests, check for vulnerabilities, and run the linter clean Remove executables and other build output flamencoManager Build Flamenco Manager with the webapp and add-on ZIP embedded flamencoManagerWithoutWebapp Only build the Flamenco Manager executable, do not rebuild the webapp flamencoWorker Build the Flamenco Worker executable generate code (OpenAPI and test mocks) generateGo Generate Go code for Flamenco Manager and Worker generateJS Generate JavaScript code for the webapp generatePy Generate Python code for the add-on govulncheck Check for known vulnerabilities. staticcheck Analyse the source code. test Run unit tests version Show which version information would be embedded in executables vet Run `go vet` webappInstallDeps Use Yarn to install the webapp's NodeJS dependencies webappStatic Build the webapp as static files that can be served ``` Co-authored-by: Mateus Abelli Reviewed-on: https://projects.blender.org/studio/flamenco/pulls/104341 --- .gitignore | 1 + Makefile | 184 +++++++------------------------ cmd/addon-packer/addon-packer.go | 167 ---------------------------- cmd/update-version/magefiles.go | 88 +++++++++++++++ cmd/update-version/main.go | 30 ++++- go.mod | 24 ++-- go.sum | 38 ++++--- mage.go | 11 ++ magefiles/addonpacker.go | 108 ++++++++++++++++++ magefiles/build.go | 134 ++++++++++++++++++++++ magefiles/check.go | 61 ++++++++++ magefiles/clean.go | 64 +++++++++++ magefiles/devserver.go | 13 +++ magefiles/generate.go | 172 +++++++++++++++++++++++++++++ magefiles/install.go | 45 ++++++++ magefiles/runner.go | 48 ++++++++ magefiles/version.go | 33 ++++++ 17 files changed, 883 insertions(+), 338 deletions(-) delete mode 100644 cmd/addon-packer/addon-packer.go create mode 100644 cmd/update-version/magefiles.go create mode 100644 mage.go create mode 100644 magefiles/addonpacker.go create mode 100644 magefiles/build.go create mode 100644 magefiles/check.go create mode 100644 magefiles/clean.go create mode 100644 magefiles/devserver.go create mode 100644 magefiles/generate.go create mode 100644 magefiles/install.go create mode 100644 magefiles/runner.go create mode 100644 magefiles/version.go diff --git a/.gitignore b/.gitignore index fd6febb2..29b56aec 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ /shaman-checkout-id-setter /stresser /job-creator +/mage /addon-packer flamenco-manager.yaml flamenco-worker.yaml diff --git a/Makefile b/Makefile index 224fa4c2..21f8eba8 100644 --- a/Makefile +++ b/Makefile @@ -23,10 +23,11 @@ ifeq (${GITHASH},dirty) GITHASH := $(shell git rev-parse --short=9 HEAD) endif -LDFLAGS := ${LDFLAGS} -X ${PKG}/internal/appinfo.ApplicationVersion=${VERSION} \ - -X ${PKG}/internal/appinfo.ApplicationGitHash=${GITHASH} \ - -X ${PKG}/internal/appinfo.ReleaseCycle=${RELEASE_CYCLE} -BUILD_FLAGS = -ldflags="${LDFLAGS}" +BUILDTOOL := mage +ifeq ($(OS),Windows_NT) + BUILDTOOL := $(BUILDTOOL).exe +endif +BUILDTOOL_PATH := ${PWD}/${BUILDTOOL} # Package name of the generated Python/JavaScript code for the Flamenco API. PY_API_PKG_NAME=flamenco.manager @@ -48,28 +49,29 @@ export CGO_ENABLED=0 all: application # Install generators and build the software. -with-deps: - go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.9.0 - go install github.com/golang/mock/mockgen@v1.6.0 +with-deps: buildtool + "${BUILDTOOL_PATH}" installGenerators $(MAKE) application -vet: - go vet ./... - go run golang.org/x/vuln/cmd/govulncheck@latest ./... +vet: buildtool + "${BUILDTOOL_PATH}" vet -application: webapp flamenco-manager flamenco-worker +application: flamenco-manager flamenco-worker -flamenco-manager: - $(MAKE) webapp-static - go build -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-manager +flamenco-manager: buildtool + "${BUILDTOOL_PATH}" flamencoManager -.PHONY: flamenco-manager-without-webapp -flamenco-manager-without-webapp: - go build -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-manager +flamenco-manager-without-webapp: buildtool + "${BUILDTOOL_PATH}" flamencoManagerWithoutWebapp -flamenco-worker: - go build -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-worker +flamenco-worker: buildtool + "${BUILDTOOL_PATH}" flamencoWorker +# Builds the buildtool itself, for faster rebuilds of Skyfill. +buildtool: ${BUILDTOOL} +${BUILDTOOL}: mage.go $(wildcard magefiles/*.go) go.mod + @echo "Building build tool $@" + @go run mage.go -compile "${BUILDTOOL_PATH}" # NOTE: these database migration commands are just for reference / debugging / # development purposes. Flamenco Manager and Worker each perform their own @@ -85,118 +87,20 @@ db-migrate-down: goose -dir ./internal/manager/persistence/migrations/ sqlite3 flamenco-manager.sqlite down .PHONY: db-migrate-status db-migrate-up db-migrate-down -.PHONY: stresser -stresser: - go build -v ${BUILD_FLAGS} ${PKG}/cmd/stresser +webapp-static: buildtool + "${BUILDTOOL_PATH}" webappStatic -.PHONY: job-creator -job-creator: - go build -v ${BUILD_FLAGS} ${PKG}/cmd/job-creator +generate: buildtool + "${BUILDTOOL_PATH}" generate -flamenco-addon.zip: addon-packer - ./addon-packer -filename ./flamenco-addon.zip +generate-go: buildtool + "${BUILDTOOL_PATH}" generateGo -addon-packer: cmd/addon-packer/addon-packer.go - go build -v ${BUILD_FLAGS} ${PKG}/cmd/addon-packer +generate-py: buildtool + "${BUILDTOOL_PATH}" generatePy -flamenco-manager_race: - CGO_ENABLED=1 go build -race -o $@ -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-manager - -flamenco-worker_race: - CGO_ENABLED=1 go build -race -o $@ -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-worker - -.PHONY: shaman-checkout-id-setter -shaman-checkout-id-setter: - go build -v ${BUILD_FLAGS} ${PKG}/cmd/shaman-checkout-id-setter - -webapp: - yarn --cwd web/app install - -webapp-static: addon-packer - $(MAKE) clean-webapp-static -# When changing the base URL, also update the line -# e.GET("/app/*", echo.WrapHandler(webAppHandler)) -# in `cmd/flamenco-manager/main.go` - MSYS2_ARG_CONV_EXCL="*" yarn --cwd web/app build --outDir ../static --base=/app/ --logLevel warn -# yarn --cwd web/app build --outDir ../static --base=/app/ --minify false - ./addon-packer -filename ${WEB_STATIC}/flamenco-addon.zip - @echo "Web app has been installed into ${WEB_STATIC}" - -generate: - $(MAKE) generate-go - $(MAKE) generate-py - $(MAKE) generate-js - -generate-go: - go generate ./pkg/api/... - go generate ./internal/... -# The generators always produce UNIX line-ends. This creates false file -# modifications with Git. Convert them to DOS line-ends to avoid this. -ifeq ($(OS),Windows_NT) - git status --porcelain | grep '^ M .*.gen.go' | cut -d' ' -f3 | xargs unix2dos --keepdate -endif - -generate-py: -# The generator doesn't consistently overwrite existing files, nor does it -# remove no-longer-generated files. - rm -rf addon/flamenco/manager - -# See https://openapi-generator.tech/docs/generators/python for the options. - java -jar addon/openapi-generator-cli.jar \ - generate \ - -i pkg/api/flamenco-openapi.yaml \ - -g python \ - -o addon/ \ - --package-name "${PY_API_PKG_NAME}" \ - --http-user-agent "Flamenco/${VERSION} (Blender add-on)" \ - -p generateSourceCodeOnly=true \ - -p projectName=Flamenco \ - -p packageVersion="${VERSION}" > .openapi-generator-py.log - -# The generator outputs files so that we can write our own tests. We don't, -# though, so it's better to just remove those placeholders. - rm -rf addon/flamenco/manager/test -# The generators always produce UNIX line-ends. This creates false file -# modifications with Git. Convert them to DOS line-ends to avoid this. -ifeq ($(OS),Windows_NT) - git status --porcelain | grep '^ M addon/flamenco/manager' | cut -d' ' -f3 | xargs unix2dos --keepdate -endif - -generate-js: -# The generator doesn't consistently overwrite existing files, nor does it -# remove no-longer-generated files. - rm -rf web/app/src/manager-api - rm -rf web/_tmp-manager-api-javascript - -# See https://openapi-generator.tech/docs/generators/javascript for the options. -# Version '0.0.0' is used as NPM doesn't like Git hashes as versions. -# -# -p modelPropertyNaming=original is needed because otherwise the generator will -# use original naming internally, but generate docs with camelCase, and then -# things don't work properly. - java -jar addon/openapi-generator-cli.jar \ - generate \ - -i pkg/api/flamenco-openapi.yaml \ - -g javascript \ - -o web/_tmp-manager-api-javascript \ - --http-user-agent "Flamenco/${VERSION} / webbrowser" \ - -p projectName=flamenco-manager \ - -p projectVersion="0.0.0" \ - -p apiPackage="${JS_API_PKG_NAME}" \ - -p disallowAdditionalPropertiesIfNotPresent=false \ - -p usePromises=true \ - -p moduleName=flamencoManager > .openapi-generator-js.log - -# Cherry-pick the generated sources, and remove everything else. -# The only relevant bit is that the generated code depends on `superagent`, -# which is listed in our `. - mv web/_tmp-manager-api-javascript/src web/app/src/manager-api - rm -rf web/_tmp-manager-api-javascript -# The generators always produce UNIX line-ends. This creates false file -# modifications with Git. Convert them to DOS line-ends to avoid this. -ifeq ($(OS),Windows_NT) - git status --porcelain | grep '^ M web/app/src/manager-api' | cut -d' ' -f3 | xargs unix2dos --keepdate -endif +generate-js: buildtool + "${BUILDTOOL_PATH}" generateJS .PHONY: update-version: @@ -213,6 +117,7 @@ update-version: addon/flamenco/__init__.py \ addon/flamenco/manager \ addon/flamenco/manager_README.md \ + magefiles/version.go \ web/app/src/manager-api \ web/project-website/data/flamenco.yaml @echo 'git tag -a -m "Tagged version ${VERSION}" v${VERSION}' @@ -237,28 +142,17 @@ swagger-ui: @echo @echo 'Now update pkg/api/static/swagger-ui/index.html to have url: "/api/openapi3.json",' -test: -# Ensure the web-static directory exists, so that `web/web_app.go` can embed something. - mkdir -p ${WEB_STATIC} - go test -short -failfast ./... +test: buildtool + "${BUILDTOOL_PATH}" test -clean: - @go clean -i -x - rm -f flamenco*-v* flamenco-manager flamenco-worker *.exe flamenco-*_race addon-packer stresser - $(MAKE) clean-webapp-static - -clean-webapp-static: -# Start with `./` to avoid horrors when WEB_STATIC is absolute (like / or /home/yourname). - rm -rf ./${WEB_STATIC} -# Make sure there is at least something to embed by Go, or it may cause some errors. - mkdir -p ./${WEB_STATIC} - touch ${WEB_STATIC}/emptyfile +clean: buildtool + "${BUILDTOOL_PATH}" clean devserver-website: go run ${HUGO_PKG} -s web/project-website serve -devserver-webapp: - yarn --cwd web/app run dev --host +devserver-webapp: buildtool + "${BUILDTOOL_PATH}" devServerWebapp deploy-website: $(MAKE) -s check-environment @@ -417,4 +311,4 @@ publish-release-packages: ${RELEASE_PACKAGE_LINUX} ${RELEASE_PACKAGE_DARWIN} ${RELEASE_PACKAGE_DARWIN_ARM64} ${RELEASE_PACKAGE_WINDOWS} ${RELEASE_PACKAGE_SHAFILE} \ ${WEBSERVER_SSH}:${WEBSERVER_ROOT}/downloads/ -.PHONY: application version flamenco-manager flamenco-worker flamenco-manager_race flamenco-worker_race webapp webapp-static generate generate-go generate-py with-deps swagger-ui list-embedded test clean clean-webapp-static +.PHONY: application version flamenco-manager flamenco-worker flamenco-manager_race flamenco-worker_race webapp webapp-static generate generate-go generate-py with-deps swagger-ui list-embedded test clean flamenco-manager-without-webapp diff --git a/cmd/addon-packer/addon-packer.go b/cmd/addon-packer/addon-packer.go deleted file mode 100644 index 5b842ae3..00000000 --- a/cmd/addon-packer/addon-packer.go +++ /dev/null @@ -1,167 +0,0 @@ -package main - -// SPDX-License-Identifier: GPL-3.0-or-later - -import ( - "archive/zip" - "compress/flate" - "flag" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "strings" - "time" - - "github.com/mattn/go-colorable" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - - "projects.blender.org/studio/flamenco/internal/appinfo" -) - -var cliArgs struct { - // Do-and-quit flags. - version bool - - // Logging level flags. - quiet, debug, trace bool - - filename string -} - -func main() { - parseCliArgs() - if cliArgs.version { - fmt.Println(appinfo.ApplicationVersion) - return - } - - output := zerolog.ConsoleWriter{Out: colorable.NewColorableStdout(), TimeFormat: time.RFC3339} - log.Logger = log.Output(output) - configLogLevel() - - outfile, err := filepath.Abs(cliArgs.filename) - if err != nil { - log.Fatal().Err(err).Str("filepath", cliArgs.filename).Msg("unable make output file path absolute") - } - - // Open the output file. - logger := log.With().Str("zipname", outfile).Logger() - logger.Info().Msg("creating ZIP file") - zipFile, err := os.Create(outfile) - if err != nil { - logger.Fatal().Err(err).Msg("error creating file") - } - defer zipFile.Close() - zipWriter := zip.NewWriter(zipFile) - - zipWriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) { - return flate.NewWriter(out, flate.BestCompression) - }) - - // CD to the addon dir to get the relative paths nice. - if err := os.Chdir("addon"); err != nil { - log.Fatal().Err(err).Msg("unable to cd to addon") - } - - basePath, err := os.Getwd() - if err != nil { - logger.Fatal().Err(err).Msg("error getting current working directory") - } - - // Copy all the files into the ZIP. - addToZip := func(path string, d fs.DirEntry, err error) error { - sublog := log.With().Str("path", path).Logger() - - if err != nil { - sublog.Error().Err(err).Msg("error received from filepath.WalkDir, aborting") - return err - } - - // Construct the path inside the ZIP file. - relpath, err := filepath.Rel(basePath, path) - if err != nil { - return fmt.Errorf("making %s relative to %s: %w", path, basePath, err) - } - - if d.IsDir() { - switch { - case filepath.Base(path) == "__pycache__": - return fs.SkipDir - case relpath == filepath.Join("flamenco", "manager", "docs"): - return fs.SkipDir - case strings.HasPrefix(filepath.Base(path), "."): - // Skip directories like .mypy_cache, etc. - return fs.SkipDir - default: - // Just recurse into this directory. - return nil - } - } - - sublog.Debug().Str("path", relpath).Msg("adding file to ZIP") - - // Read the file's contents. These are just Python files and maybe a Wheel, - // nothing huge. - contents, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("reading %s: %w", path, err) - } - - // Write into the ZIP file. - fileInZip, err := zipWriter.Create(relpath) - if err != nil { - return fmt.Errorf("creating %s in ZIP: %w", relpath, err) - } - _, err = fileInZip.Write(contents) - if err != nil { - return fmt.Errorf("writing to %s in ZIP: %w", relpath, err) - } - - return nil - } - - logger.Debug().Str("cwd", basePath).Msg("walking directory") - - if err := filepath.WalkDir(filepath.Join(basePath, "flamenco"), addToZip); err != nil { - logger.Fatal().Err(err).Msg("error filling ZIP file") - } - - comment := fmt.Sprintf("%s add-on for Blender, version %s", - appinfo.ApplicationName, - appinfo.ApplicationVersion, - ) - if err := zipWriter.SetComment(comment); err != nil { - logger.Fatal().Err(err).Msg("error setting ZIP comment") - } - - if err := zipWriter.Close(); err != nil { - logger.Fatal().Err(err).Msg("error closing ZIP file") - } -} - -func parseCliArgs() { - flag.BoolVar(&cliArgs.version, "version", false, "Shows the application version, then exits.") - flag.BoolVar(&cliArgs.quiet, "quiet", false, "Only log warning-level and worse.") - flag.BoolVar(&cliArgs.debug, "debug", false, "Enable debug-level logging.") - flag.BoolVar(&cliArgs.trace, "trace", false, "Enable trace-level logging.") - flag.StringVar(&cliArgs.filename, "filename", "web/static/flamenco-addon.zip", "Filename to save the add-on to.") - flag.Parse() -} - -func configLogLevel() { - var logLevel zerolog.Level - switch { - case cliArgs.trace: - logLevel = zerolog.TraceLevel - case cliArgs.debug: - logLevel = zerolog.DebugLevel - case cliArgs.quiet: - logLevel = zerolog.WarnLevel - default: - logLevel = zerolog.InfoLevel - } - zerolog.SetGlobalLevel(logLevel) -} diff --git a/cmd/update-version/magefiles.go b/cmd/update-version/magefiles.go new file mode 100644 index 00000000..8171e498 --- /dev/null +++ b/cmd/update-version/magefiles.go @@ -0,0 +1,88 @@ +package main + +import ( + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "path/filepath" + + "github.com/rs/zerolog/log" +) + +const mageFile = "magefiles/version.go" + +// updateMagefiles changes the version number in the Mage files. +// Returns whether the file actually changed. +func updateMagefiles() bool { + logger := log.With().Str("filename", mageFile).Logger() + + // Parse the mage file as AST. + fset := token.NewFileSet() + astFile, err := parser.ParseFile(fset, mageFile, nil, parser.SkipObjectResolution|parser.ParseComments) + if err != nil { + logger.Fatal().Err(err).Msgf("could not update mage version file") + return false + } + + // Perform replacements on the AST. + replacements := map[string]string{ + "version": cliArgs.newVersion, + "releaseCycle": releaseCycle, + } + + var ( + lastIdent *ast.Ident // Last-seen identifier. + anyFieldChanged bool + ) + + ast.Inspect(astFile, func(node ast.Node) bool { + switch x := node.(type) { + case *ast.Ident: + lastIdent = x + + case *ast.BasicLit: + replacement, ok := replacements[lastIdent.Name] + if ok { + newValue := fmt.Sprintf("%q", replacement) + if x.Value != newValue { + logger.Info(). + Str("old", x.Value). + Str("new", newValue). + Msg("updating mage version file") + x.Value = newValue + anyFieldChanged = true + } + } + } + + return true + }) + + // Open a temporary file for writing. + mageDir := filepath.Dir(mageFile) + writer, err := os.CreateTemp(mageDir, filepath.Base(mageFile)+"*.go") + if err != nil { + log.Fatal().Err(err).Msgf("cannot create file in %s", mageDir) + } + defer writer.Close() + + // Write the altered AST to the temp file. + if err := format.Node(writer, fset, astFile); err != nil { + log.Fatal().Err(err).Msgf("cannot write updated version of %s to %s", mageFile, writer.Name()) + } + + // Close the file. + if err := writer.Close(); err != nil { + log.Fatal().Err(err).Msgf("cannot close %s", writer.Name()) + } + + // Overwrite the original mage file with the temp file. + if err := os.Rename(writer.Name(), mageFile); err != nil { + log.Fatal().Err(err).Msgf("cannot rename %s to %s", writer.Name(), mageFile) + } + + return anyFieldChanged +} diff --git a/cmd/update-version/main.go b/cmd/update-version/main.go index 01de5567..70f9f0fe 100644 --- a/cmd/update-version/main.go +++ b/cmd/update-version/main.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "os" + "strings" "time" "github.com/mattn/go-colorable" @@ -13,13 +14,18 @@ import ( "github.com/rs/zerolog/log" ) -var cliArgs struct { - // Logging level flags. - quiet, debug, trace bool +// Global variables used by the updateXXX() functions. +var ( + cliArgs struct { + // Logging level flags. + quiet, debug, trace bool - newVersion string - updateMakefile bool -} + newVersion string + updateMakefile bool + } + + releaseCycle string +) func main() { parseCliArgs() @@ -29,11 +35,23 @@ func main() { log.Info().Str("version", cliArgs.newVersion).Msg("updating Flamenco version") + switch { + case strings.Contains(cliArgs.newVersion, "alpha"), strings.Contains(cliArgs.newVersion, "dev"): + releaseCycle = "alpha" + case strings.Contains(cliArgs.newVersion, "beta"): + releaseCycle = "beta" + case strings.Contains(cliArgs.newVersion, "rc"): + releaseCycle = "rc" + default: + releaseCycle = "release" + } + var anyFileWasChanged bool if cliArgs.updateMakefile { anyFileWasChanged = updateMakefile() || anyFileWasChanged } anyFileWasChanged = updateAddon() || anyFileWasChanged + anyFileWasChanged = updateMagefiles() || anyFileWasChanged if !anyFileWasChanged { log.Warn().Msg("nothing changed") diff --git a/go.mod b/go.mod index cd1c7e2f..7ac3d647 100644 --- a/go.mod +++ b/go.mod @@ -19,22 +19,27 @@ require ( github.com/google/uuid v1.5.0 github.com/graarh/golang-socketio v0.0.0-20170510162725-2c44953b9b5f github.com/labstack/echo/v4 v4.9.1 + github.com/magefile/mage v1.15.0 github.com/mattn/go-colorable v0.1.12 - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/pressly/goose/v3 v3.15.1 github.com/rs/zerolog v1.26.1 github.com/stretchr/testify v1.8.4 github.com/zcalusic/sysinfo v1.0.1 github.com/ziflex/lecho/v3 v3.1.0 - golang.org/x/crypto v0.23.0 + golang.org/x/crypto v0.25.0 golang.org/x/image v0.18.0 - golang.org/x/net v0.25.0 - golang.org/x/sys v0.20.0 + golang.org/x/net v0.27.0 + golang.org/x/sync v0.7.0 + golang.org/x/sys v0.22.0 + golang.org/x/vuln v1.1.3 gopkg.in/yaml.v2 v2.4.0 + honnef.co/go/tools v0.5.1 modernc.org/sqlite v1.28.0 ) require ( + github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -54,11 +59,12 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/sync v0.7.0 // indirect + golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.23.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/uint128 v1.3.0 // indirect modernc.org/cc/v3 v3.41.0 // indirect @@ -70,3 +76,7 @@ require ( modernc.org/strutil v1.2.0 // indirect modernc.org/token v1.1.0 // indirect ) + +// Replace staticcheck release with a specific revision of their `main` branch, +// so that it includes my PR https://github.com/dominikh/go-tools/pull/1597 +replace honnef.co/go/tools v0.5.1 => honnef.co/go/tools v0.0.0-20240920144234-9f4b51e3ab5a diff --git a/go.sum b/go.sum index efbc1c53..2a7c1825 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs= +github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= @@ -115,6 +117,8 @@ github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= github.com/lestrrat-go/jwx v1.2.7/go.mod h1:bw24IXWbavc0R2RsOtpXL7RtMyP589yZ1+L7kd09ZGA= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= @@ -138,8 +142,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -191,8 +195,10 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ= +golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= @@ -200,8 +206,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -211,8 +217,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -235,7 +241,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -247,9 +252,12 @@ golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7 h1:FemxDzfMUcK2f3YY4H+05K9CDzbSVr2+q/JKN45pey0= +golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -271,8 +279,10 @@ golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/vuln v1.1.3 h1:NPGnvPOTgnjBc9HTaUx+nj+EaUYxl5SJOWqaDYGaFYw= +golang.org/x/vuln v1.1.3/go.mod h1:7Le6Fadm5FOqE9C926BCD0g12NWyhg7cxV4BwcPFuNY= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -294,6 +304,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20240920144234-9f4b51e3ab5a h1:P0jdAtRz/UFVIXrgf9EdWApQJnbQrmsUwyARz4FF+D8= +honnef.co/go/tools v0.0.0-20240920144234-9f4b51e3ab5a/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs= lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q= diff --git a/mage.go b/mage.go new file mode 100644 index 00000000..6a3b958e --- /dev/null +++ b/mage.go @@ -0,0 +1,11 @@ +//go:build ignore + +package main + +import ( + "os" + + "github.com/magefile/mage/mage" +) + +func main() { os.Exit(mage.Main()) } diff --git a/magefiles/addonpacker.go b/magefiles/addonpacker.go new file mode 100644 index 00000000..ee78417e --- /dev/null +++ b/magefiles/addonpacker.go @@ -0,0 +1,108 @@ +//go:build mage + +package main + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "archive/zip" + "compress/flate" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "projects.blender.org/studio/flamenco/internal/appinfo" +) + +func packAddon(filename string) error { + outfile, err := filepath.Abs(filename) + if err != nil { + return fmt.Errorf("unable make output file path absolute: %w", err) + } + + // Open the output file. + zipFile, err := os.Create(outfile) + if err != nil { + return fmt.Errorf("error creating file %s: %w", outfile, err) + } + defer zipFile.Close() + + zipWriter := zip.NewWriter(zipFile) + defer zipWriter.Close() + + zipWriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) { + return flate.NewWriter(out, flate.BestCompression) + }) + + basePath, err := filepath.Abs("./addon") // os.Getwd() + if err != nil { + return fmt.Errorf("error getting current working directory: %w", err) + } + + // Copy all the files into the ZIP. + addToZip := func(path string, d fs.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("error received from filepath.WalkDir: %w", err) + } + + // Construct the path inside the ZIP file. + relpath, err := filepath.Rel(basePath, path) + if err != nil { + return fmt.Errorf("making %s relative to %s: %w", path, basePath, err) + } + + if d.IsDir() { + switch { + case filepath.Base(path) == "__pycache__": + return fs.SkipDir + case relpath == filepath.Join("flamenco", "manager", "docs"): + return fs.SkipDir + case strings.HasPrefix(filepath.Base(path), "."): + // Skip directories like .mypy_cache, etc. + return fs.SkipDir + default: + // Just recurse into this directory. + return nil + } + } + + // Read the file's contents. These are just Python files and maybe a Wheel, + // nothing huge. + contents, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + + // Write into the ZIP file. + fileInZip, err := zipWriter.Create(relpath) + if err != nil { + return fmt.Errorf("creating %s in ZIP: %w", relpath, err) + } + _, err = fileInZip.Write(contents) + if err != nil { + return fmt.Errorf("writing to %s in ZIP: %w", relpath, err) + } + + return nil + } + + if err := filepath.WalkDir(filepath.Join(basePath, "flamenco"), addToZip); err != nil { + return fmt.Errorf("error filling ZIP file: %w", err) + } + + comment := fmt.Sprintf("%s add-on for Blender, version %s", + appinfo.ApplicationName, + appinfo.ApplicationVersion, + ) + if err := zipWriter.SetComment(comment); err != nil { + return fmt.Errorf("error setting ZIP comment: %w", err) + } + + if err := zipWriter.Close(); err != nil { + return fmt.Errorf("error closing ZIP file: %w", err) + } + return nil +} diff --git a/magefiles/build.go b/magefiles/build.go new file mode 100644 index 00000000..63438d47 --- /dev/null +++ b/magefiles/build.go @@ -0,0 +1,134 @@ +//go:build mage + +package main + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "fmt" + "path/filepath" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "github.com/magefile/mage/target" +) + +const ( + goPkg = "projects.blender.org/studio/flamenco" +) + +var ( + // The directory that will contain the built webapp files, and some other + // files that will be served as static files by the Flamenco Manager web + // server. + webStatic = filepath.Join("web", "static") +) + +// Build Flamenco Manager and Flamenco Worker, including the webapp and the add-on +func Build() { + mg.Deps(FlamencoManager, FlamencoWorker) +} + +// Build Flamenco Manager with the webapp and add-on ZIP embedded +func FlamencoManager() error { + mg.Deps(WebappStatic) + mg.Deps(flamencoManager) + return nil +} + +// Only build the Flamenco Manager executable, do not rebuild the webapp +func FlamencoManagerWithoutWebapp() error { + mg.Deps(flamencoManager) + return nil +} + +func flamencoManager() error { + return build("./cmd/flamenco-manager") +} + +// Build the Flamenco Worker executable +func FlamencoWorker() error { + return build("./cmd/flamenco-worker") +} + +// Build the webapp as static files that can be served +func WebappStatic() error { + runInstall, err := target.Dir("web/app/node_modules") + if err != nil { + return err + } + if runInstall { + mg.SerialDeps(WebappInstallDeps) + } + if err := cleanWebappStatic(); err != nil { + return err + } + + env := map[string]string{ + "MSYS2_ARG_CONV_EXCL": "*", + } + + // When changing the base URL, also update the line + // e.GET("/app/*", echo.WrapHandler(webAppHandler)) + // in `cmd/flamenco-manager/main.go` + err = sh.RunWithV(env, + "yarn", + "--cwd", "web/app", + "build", + "--outDir", "../static", + "--base=/app/", + "--logLevel", "warn", + // For debugging you can add: + // "--minify", "false", + ) + if err != nil { + return err + } + + fmt.Printf("Web app has been installed into %s\n", webStatic) + + // Build the add-on ZIP as it's part of the static web files. + zipPath := filepath.Join(webStatic, "flamenco3-addon.zip") + return packAddon(zipPath) +} + +// Use Yarn to install the webapp's NodeJS dependencies +func WebappInstallDeps() error { + env := map[string]string{ + "MSYS2_ARG_CONV_EXCL": "*", + } + return sh.RunWithV(env, + "yarn", + "--cwd", "web/app", + "install", + ) +} + +func build(exePackage string) error { + flags, err := buildFlags() + if err != nil { + return err + } + + args := []string{"build", "-v"} + args = append(args, flags...) + args = append(args, exePackage) + return sh.RunV(mg.GoCmd(), args...) +} + +func buildFlags() ([]string, error) { + hash, err := gitHash() + if err != nil { + return nil, err + } + + ldflags := "" + + fmt.Sprintf(" -X %s/internal/appinfo.ApplicationVersion=%s", goPkg, version) + + fmt.Sprintf(" -X %s/internal/appinfo.ApplicationGitHash=%s", goPkg, hash) + + fmt.Sprintf(" -X %s/internal/appinfo.ReleaseCycle=%s", goPkg, releaseCycle) + + flags := []string{ + "-ldflags=" + ldflags, + } + return flags, nil +} diff --git a/magefiles/check.go b/magefiles/check.go new file mode 100644 index 00000000..4e179653 --- /dev/null +++ b/magefiles/check.go @@ -0,0 +1,61 @@ +//go:build mage + +package main + +import ( + "context" + "errors" + "fmt" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "golang.org/x/vuln/scan" + "honnef.co/go/tools/lintcmd" + lintcmdversion "honnef.co/go/tools/lintcmd/version" + "honnef.co/go/tools/simple" + "honnef.co/go/tools/staticcheck" + "honnef.co/go/tools/stylecheck" + "honnef.co/go/tools/unused" +) + +// Run unit tests, check for vulnerabilities, and run the linter +func Check(ctx context.Context) { + mg.CtxDeps(ctx, Test, Govulncheck, Staticcheck, Vet) +} + +// Run unit tests +func Test(ctx context.Context) error { + return sh.RunV(mg.GoCmd(), "test", "-short", "-failfast", "./...") +} + +// Check for known vulnerabilities. +func Govulncheck(ctx context.Context) error { + cmd := scan.Command(ctx, "./...") + if err := cmd.Start(); err != nil { + return err + } + return cmd.Wait() +} + +// Analyse the source code. +func Staticcheck() error { + cmd := lintcmd.NewCommand("staticcheck") + cmd.SetVersion(lintcmdversion.Version, lintcmdversion.MachineVersion) + cmd.ParseFlags([]string{"./..."}) + cmd.AddAnalyzers(simple.Analyzers...) + cmd.AddAnalyzers(staticcheck.Analyzers...) + cmd.AddAnalyzers(stylecheck.Analyzers...) + cmd.AddAnalyzers(unused.Analyzer) + + exitCode := cmd.Execute() + if exitCode != 0 { + return errors.New("staticcheck failed") + } + fmt.Println("staticcheck ok") + return nil +} + +// Run `go vet` +func Vet() error { + return sh.RunV(mg.GoCmd(), "vet", "./...") +} diff --git a/magefiles/clean.go b/magefiles/clean.go new file mode 100644 index 00000000..659aaa6a --- /dev/null +++ b/magefiles/clean.go @@ -0,0 +1,64 @@ +//go:build mage + +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/magefile/mage/sh" +) + +// Remove executables and other build output +func Clean() error { + if err := cleanWebappStatic(); err != nil { + return err + } + + if err := sh.Run("go", "clean"); err != nil { + return err + } + + if err := rm( + "flamenco-manager", "flamenco-manager.exe", + "flamenco-manager_race", "flamenco-manager_race.exe", + "flamenco-worker", "flamenco-worker.exe", + "flamenco-worker_race", "flamenco-worker_race.exe", + ); err != nil { + return err + } + return nil +} + +func cleanWebappStatic() error { + // Just a simple heuristic to avoid deleting things like "/" or "C:\" + if len(webStatic) < 4 { + panic(fmt.Sprintf("webStatic path is too short, I don't trust it: %q", webStatic)) + } + + if err := sh.Rm(webStatic); err != nil { + return fmt.Errorf("unable to remove old web static dir %q: %w", webStatic, err) + } + if err := os.MkdirAll(webStatic, os.ModePerm); err != nil { + return fmt.Errorf("unable to create web static dir %q: %w", webStatic, err) + } + + // Make sure there is at least something to embed by Go, or it may cause some + // errors. This is done in the 'clean' function so that the Go code can be + // built before building the webapp. + emptyfile := filepath.Join(webStatic, "emptyfile") + if err := os.WriteFile(emptyfile, []byte{}, os.ModePerm); err != nil { + return err + } + return nil +} + +func rm(path ...string) error { + for _, p := range path { + if err := sh.Rm(p); err != nil { + return err + } + } + return nil +} diff --git a/magefiles/devserver.go b/magefiles/devserver.go new file mode 100644 index 00000000..d63c1478 --- /dev/null +++ b/magefiles/devserver.go @@ -0,0 +1,13 @@ +//go:build mage + +package main + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "github.com/magefile/mage/sh" +) + +func DevServerWebapp() error { + return sh.RunV("yarn", "--cwd", "web/app", "run", "dev", "--host") +} diff --git a/magefiles/generate.go b/magefiles/generate.go new file mode 100644 index 00000000..11d8cacd --- /dev/null +++ b/magefiles/generate.go @@ -0,0 +1,172 @@ +//go:build mage + +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" +) + +// Generate code (OpenAPI and test mocks) +func Generate() { + mg.Deps(GenerateGo, GeneratePy, GenerateJS) +} + +// Generate Go code for Flamenco Manager and Worker +func GenerateGo(ctx context.Context) error { + r := NewRunner(ctx) + r.Run(mg.GoCmd(), "generate", "./pkg/api/...") + r.Run(mg.GoCmd(), "generate", "./internal/...") + if err := r.Wait(); err != nil { + return err + } + + // The generators always produce UNIX line-ends. This creates false file + // modifications with Git. Convert them to DOS line-ends to avoid this. + if runtime.GOOS == "windows" { + unix2dosModifiedFiles(".gen.go$") + } + return nil +} + +// Generate Python code for the add-on +func GeneratePy() error { + // The generator doesn't consistently overwrite existing files, nor does it + // remove no-longer-generated files. + sh.Rm("addon/flamenco/manager") + + // See https://openapi-generator.tech/docs/generators/python for the options. + err := sh.Run("java", + "-jar", "addon/openapi-generator-cli.jar", + "generate", + "-i", "pkg/api/flamenco-openapi.yaml", + "-g", "python", + "-o", "addon/", + "--package-name", "flamenco.manager", + "--http-user-agent", fmt.Sprintf("Flamenco/%s (Blender add-on)", version), + "-p", "generateSourceCodeOnly=true", + "-p", "projectName=Flamenco", + "-p", fmt.Sprintf("packageVersion=%s", version)) + if err != nil { + return err + } + + // The generator outputs files so that we can write our own tests. We don't, + // though, so it's better to just remove those placeholders. + sh.Rm("addon/flamenco/manager/test") + + // The generators always produce UNIX line-ends. This creates false file + // modifications with Git. Convert them to DOS line-ends to avoid this. + if runtime.GOOS == "windows" { + unix2dosModifiedFiles("addon/flamenco/manager") + } + + return nil +} + +// Generate JavaScript code for the webapp +func GenerateJS() error { + const ( + jsOutDir = "web/app/src/manager-api" + jsTempDir = "web/_tmp-manager-api-javascript" + ) + sh.Rm(jsOutDir) + sh.Rm(jsTempDir) + + // See https://openapi-generator.tech/docs/generators/javascript for the options. + // Version '0.0.0' is used as NPM doesn't like Git hashes as versions. + // + // -p modelPropertyNaming=original is needed because otherwise the generator will + // use original naming internally, but generate docs with camelCase, and then + // things don't work properly. + err := sh.Run("java", + "-jar", "addon/openapi-generator-cli.jar", + "generate", + "-i", "pkg/api/flamenco-openapi.yaml", + "-g", "javascript", + "-o", jsTempDir, + "--http-user-agent", fmt.Sprintf("Flamenco/%s / webbrowser", version), + "-p", "projectName=flamenco-manager", + "-p", "projectVersion=0.0.0", + "-p", "apiPackage=manager", + "-p", "disallowAdditionalPropertiesIfNotPresent=false", + "-p", "usePromises=true", + "-p", "moduleName=flamencoManager") + if err != nil { + return err + } + + // Cherry-pick the generated sources, and remove everything else. + if err := os.Rename(filepath.Join(jsTempDir, "src"), jsOutDir); err != nil { + return err + } + sh.Rm(jsTempDir) + + if runtime.GOOS == "windows" { + unix2dosModifiedFiles(jsOutDir) + } + + return nil +} + +// unix2dosModifiedFiles changes line ends in files Git considers modified that match the given pattern. +func unix2dosModifiedFiles(pattern string) { + // Get modified files from Git. Expected lines like: + // + // M pkg/api/openapi_client.gen.go + // M pkg/api/openapi_server.gen.go + // M pkg/api/openapi_spec.gen.go + // M pkg/api/openapi_types.gen.go + // ?? magefiles/generate.go + + gitStatus, err := sh.Output("git", "status", "--porcelain") + if err != nil { + panic(fmt.Sprintf("error running 'git status': %s", err)) + } + + // Construct a list of modified files that match the pattern. + patternRe := regexp.MustCompile(pattern) + modified := []string{} + for _, line := range strings.Split(gitStatus, "\n") { + if line[0:3] != " M " { + continue + } + if !patternRe.MatchString(line[3:]) { + continue + } + modified = append(modified, line[3:]) + } + + // Run unix2dos on all found files. + for _, path := range modified { + unix2dos(path) + } +} + +func unix2dos(filename string) { + if mg.Verbose() { + fmt.Printf("unix2dos %s\n", filename) + } + + // TODO: rewrite to stream the data instead of reading, copying, and writing + // everything. + contents, err := os.ReadFile(filename) + if err != nil { + panic(fmt.Sprintf("error converting UNIX to DOS line ends: %v", err)) + } + + lines := bytes.Split(contents, []byte("\n")) + err = os.WriteFile(filename, bytes.Join(lines, []byte("\r\n")), os.ModePerm) + if err != nil { + panic(fmt.Sprintf("error writing DOS line ends: %v", err)) + } +} diff --git a/magefiles/install.go b/magefiles/install.go new file mode 100644 index 00000000..a90cb668 --- /dev/null +++ b/magefiles/install.go @@ -0,0 +1,45 @@ +//go:build mage + +package main + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "context" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" +) + +var ( + generators = []string{ + "github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.9.0", + "github.com/golang/mock/mockgen@v1.6.0", + } +) + +// Install build-time dependencies: code generators and NodeJS dependencies. +func InstallDeps() { + mg.SerialDeps(InstallGenerators, InstallDepsWebapp) +} + +// Install code generators. +func InstallGenerators(ctx context.Context) error { + r := NewRunner(ctx) + for _, pkg := range generators { + r.Run(mg.GoCmd(), "install", pkg) + } + return r.Wait() +} + +// Use Yarn to install the webapp's NodeJS dependencies +func InstallDepsWebapp() error { + env := map[string]string{ + "MSYS2_ARG_CONV_EXCL": "*", + } + return sh.RunWithV(env, + "yarn", + "--cwd", "web/app", + "install", + ) +} diff --git a/magefiles/runner.go b/magefiles/runner.go new file mode 100644 index 00000000..5f02017d --- /dev/null +++ b/magefiles/runner.go @@ -0,0 +1,48 @@ +//go:build mage + +package main + +import ( + "context" + + "github.com/magefile/mage/sh" + "golang.org/x/sync/errgroup" +) + +// Runner allows running a group of commands sequentially, stopping at the first +// failure. +// See https://github.com/magefile/mage/issues/455 for the feature request +// to include this in Mage. +type Runner struct { + group *errgroup.Group + ctx context.Context +} + +// NewRunner constructs a new runner that's bound to the given context. If the +// context is done, no new command will be executed. It does NOT abort an +// already-running command. +func NewRunner(ctx context.Context) *Runner { + group, groupctx := errgroup.WithContext(ctx) + group.SetLimit(1) + + return &Runner{ + group: group, + ctx: groupctx, + } +} + +// Run the given command. +// This only runs a command if no previous command has failed yet. +func (r *Runner) Run(cmd string, args ...string) { + r.group.Go(func() error { + if err := r.ctx.Err(); err != nil { + return err + } + return sh.RunV(cmd, args...) + }) +} + +// Wait for the commands to finish running, and return any error. +func (r *Runner) Wait() error { + return r.group.Wait() +} diff --git a/magefiles/version.go b/magefiles/version.go new file mode 100644 index 00000000..ff5ef66e --- /dev/null +++ b/magefiles/version.go @@ -0,0 +1,33 @@ +//go:build mage + +package main + +import ( + "fmt" + + "github.com/magefile/mage/sh" +) + +// To update the version number in all the relevant places, update the VERSION +// variable below and run `mage update-version`. +const ( + version = "3.6-alpha5" + releaseCycle = "alpha" +) + +func gitHash() (string, error) { + return sh.Output("git", "rev-parse", "--short", "HEAD") +} + +// Show which version information would be embedded in executables +func Version() error { + fmt.Printf("Package : %s\n", goPkg) + fmt.Printf("Version : %s\n", version) + + hash, err := gitHash() + if err != nil { + return err + } + fmt.Printf("Git Hash : %s\n", hash) + return nil +}