Build with Magefile (#104341)

Convert most of the code in `Makefile` to [Magefile](https://magefile.org/):

This makes it possible to build Flamenco without `make` (and the POSIX environment/commands it expect) by running:

```bash
$ go run mage.go webappInstallDeps  # Only on the first build.
$ go run mage.go build
```

More efficient builds are possible with other commands, and some release-related commands still require `make`. At least the barrier to entry should be considerably lower (compared to having to install Make + Cygwin/MSYS2 on Windows).

Fix: #102633

This does not port the building of release packages, so it doesn't address #102671.

### Main Targets

| Target   | Description                                                                     |
|----------|---------------------------------------------------------------------------------|
| build    | Build Flamenco Manager and Flamenco Worker, including the webapp and the add-on |
| check    | Run unit tests, check for vulnerabilities, and run the linter                   |
| clean    | Remove executables and other build output                                       |
| generate | Generate code (OpenAPI and test mocks)                                          |

### All Targets

Get these via `go run mage.go -l`:

```
Targets:
  build                           Flamenco Manager and Flamenco Worker, including the webapp and the add-on
  check                           Run unit tests, check for vulnerabilities, and run the linter
  clean                           Remove executables and other build output
  flamencoManager                 Build Flamenco Manager with the webapp and add-on ZIP embedded
  flamencoManagerWithoutWebapp    Only build the Flamenco Manager executable, do not rebuild the webapp
  flamencoWorker                  Build the Flamenco Worker executable
  generate                        code (OpenAPI and test mocks)
  generateGo                      Generate Go code for Flamenco Manager and Worker
  generateJS                      Generate JavaScript code for the webapp
  generatePy                      Generate Python code for the add-on
  govulncheck                     Check for known vulnerabilities.
  staticcheck                     Analyse the source code.
  test                            Run unit tests
  version                         Show which version information would be embedded in executables
  vet                             Run `go vet`
  webappInstallDeps               Use Yarn to install the webapp's NodeJS dependencies
  webappStatic                    Build the webapp as static files that can be served
```

Co-authored-by: Mateus Abelli <mateusabelli@gmail.com>
Reviewed-on: https://projects.blender.org/studio/flamenco/pulls/104341
This commit is contained in:
Sybren A. Stüvel 2024-10-04 21:59:44 +02:00
parent d61f6c9e14
commit 5f37bcb629
17 changed files with 883 additions and 338 deletions

1
.gitignore vendored
View File

@ -14,6 +14,7 @@
/shaman-checkout-id-setter
/stresser
/job-creator
/mage
/addon-packer
flamenco-manager.yaml
flamenco-worker.yaml

184
Makefile
View File

@ -23,10 +23,11 @@ ifeq (${GITHASH},dirty)
GITHASH := $(shell git rev-parse --short=9 HEAD)
endif
LDFLAGS := ${LDFLAGS} -X ${PKG}/internal/appinfo.ApplicationVersion=${VERSION} \
-X ${PKG}/internal/appinfo.ApplicationGitHash=${GITHASH} \
-X ${PKG}/internal/appinfo.ReleaseCycle=${RELEASE_CYCLE}
BUILD_FLAGS = -ldflags="${LDFLAGS}"
BUILDTOOL := mage
ifeq ($(OS),Windows_NT)
BUILDTOOL := $(BUILDTOOL).exe
endif
BUILDTOOL_PATH := ${PWD}/${BUILDTOOL}
# Package name of the generated Python/JavaScript code for the Flamenco API.
PY_API_PKG_NAME=flamenco.manager
@ -48,28 +49,29 @@ export CGO_ENABLED=0
all: application
# Install generators and build the software.
with-deps:
go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.9.0
go install github.com/golang/mock/mockgen@v1.6.0
with-deps: buildtool
"${BUILDTOOL_PATH}" installGenerators
$(MAKE) application
vet:
go vet ./...
go run golang.org/x/vuln/cmd/govulncheck@latest ./...
vet: buildtool
"${BUILDTOOL_PATH}" vet
application: webapp flamenco-manager flamenco-worker
application: flamenco-manager flamenco-worker
flamenco-manager:
$(MAKE) webapp-static
go build -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-manager
flamenco-manager: buildtool
"${BUILDTOOL_PATH}" flamencoManager
.PHONY: flamenco-manager-without-webapp
flamenco-manager-without-webapp:
go build -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-manager
flamenco-manager-without-webapp: buildtool
"${BUILDTOOL_PATH}" flamencoManagerWithoutWebapp
flamenco-worker:
go build -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-worker
flamenco-worker: buildtool
"${BUILDTOOL_PATH}" flamencoWorker
# Builds the buildtool itself, for faster rebuilds of Skyfill.
buildtool: ${BUILDTOOL}
${BUILDTOOL}: mage.go $(wildcard magefiles/*.go) go.mod
@echo "Building build tool $@"
@go run mage.go -compile "${BUILDTOOL_PATH}"
# NOTE: these database migration commands are just for reference / debugging /
# development purposes. Flamenco Manager and Worker each perform their own
@ -85,118 +87,20 @@ db-migrate-down:
goose -dir ./internal/manager/persistence/migrations/ sqlite3 flamenco-manager.sqlite down
.PHONY: db-migrate-status db-migrate-up db-migrate-down
.PHONY: stresser
stresser:
go build -v ${BUILD_FLAGS} ${PKG}/cmd/stresser
webapp-static: buildtool
"${BUILDTOOL_PATH}" webappStatic
.PHONY: job-creator
job-creator:
go build -v ${BUILD_FLAGS} ${PKG}/cmd/job-creator
generate: buildtool
"${BUILDTOOL_PATH}" generate
flamenco-addon.zip: addon-packer
./addon-packer -filename ./flamenco-addon.zip
generate-go: buildtool
"${BUILDTOOL_PATH}" generateGo
addon-packer: cmd/addon-packer/addon-packer.go
go build -v ${BUILD_FLAGS} ${PKG}/cmd/addon-packer
generate-py: buildtool
"${BUILDTOOL_PATH}" generatePy
flamenco-manager_race:
CGO_ENABLED=1 go build -race -o $@ -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-manager
flamenco-worker_race:
CGO_ENABLED=1 go build -race -o $@ -v ${BUILD_FLAGS} ${PKG}/cmd/flamenco-worker
.PHONY: shaman-checkout-id-setter
shaman-checkout-id-setter:
go build -v ${BUILD_FLAGS} ${PKG}/cmd/shaman-checkout-id-setter
webapp:
yarn --cwd web/app install
webapp-static: addon-packer
$(MAKE) clean-webapp-static
# When changing the base URL, also update the line
# e.GET("/app/*", echo.WrapHandler(webAppHandler))
# in `cmd/flamenco-manager/main.go`
MSYS2_ARG_CONV_EXCL="*" yarn --cwd web/app build --outDir ../static --base=/app/ --logLevel warn
# yarn --cwd web/app build --outDir ../static --base=/app/ --minify false
./addon-packer -filename ${WEB_STATIC}/flamenco-addon.zip
@echo "Web app has been installed into ${WEB_STATIC}"
generate:
$(MAKE) generate-go
$(MAKE) generate-py
$(MAKE) generate-js
generate-go:
go generate ./pkg/api/...
go generate ./internal/...
# The generators always produce UNIX line-ends. This creates false file
# modifications with Git. Convert them to DOS line-ends to avoid this.
ifeq ($(OS),Windows_NT)
git status --porcelain | grep '^ M .*.gen.go' | cut -d' ' -f3 | xargs unix2dos --keepdate
endif
generate-py:
# The generator doesn't consistently overwrite existing files, nor does it
# remove no-longer-generated files.
rm -rf addon/flamenco/manager
# See https://openapi-generator.tech/docs/generators/python for the options.
java -jar addon/openapi-generator-cli.jar \
generate \
-i pkg/api/flamenco-openapi.yaml \
-g python \
-o addon/ \
--package-name "${PY_API_PKG_NAME}" \
--http-user-agent "Flamenco/${VERSION} (Blender add-on)" \
-p generateSourceCodeOnly=true \
-p projectName=Flamenco \
-p packageVersion="${VERSION}" > .openapi-generator-py.log
# The generator outputs files so that we can write our own tests. We don't,
# though, so it's better to just remove those placeholders.
rm -rf addon/flamenco/manager/test
# The generators always produce UNIX line-ends. This creates false file
# modifications with Git. Convert them to DOS line-ends to avoid this.
ifeq ($(OS),Windows_NT)
git status --porcelain | grep '^ M addon/flamenco/manager' | cut -d' ' -f3 | xargs unix2dos --keepdate
endif
generate-js:
# The generator doesn't consistently overwrite existing files, nor does it
# remove no-longer-generated files.
rm -rf web/app/src/manager-api
rm -rf web/_tmp-manager-api-javascript
# See https://openapi-generator.tech/docs/generators/javascript for the options.
# Version '0.0.0' is used as NPM doesn't like Git hashes as versions.
#
# -p modelPropertyNaming=original is needed because otherwise the generator will
# use original naming internally, but generate docs with camelCase, and then
# things don't work properly.
java -jar addon/openapi-generator-cli.jar \
generate \
-i pkg/api/flamenco-openapi.yaml \
-g javascript \
-o web/_tmp-manager-api-javascript \
--http-user-agent "Flamenco/${VERSION} / webbrowser" \
-p projectName=flamenco-manager \
-p projectVersion="0.0.0" \
-p apiPackage="${JS_API_PKG_NAME}" \
-p disallowAdditionalPropertiesIfNotPresent=false \
-p usePromises=true \
-p moduleName=flamencoManager > .openapi-generator-js.log
# Cherry-pick the generated sources, and remove everything else.
# The only relevant bit is that the generated code depends on `superagent`,
# which is listed in our `.
mv web/_tmp-manager-api-javascript/src web/app/src/manager-api
rm -rf web/_tmp-manager-api-javascript
# The generators always produce UNIX line-ends. This creates false file
# modifications with Git. Convert them to DOS line-ends to avoid this.
ifeq ($(OS),Windows_NT)
git status --porcelain | grep '^ M web/app/src/manager-api' | cut -d' ' -f3 | xargs unix2dos --keepdate
endif
generate-js: buildtool
"${BUILDTOOL_PATH}" generateJS
.PHONY:
update-version:
@ -213,6 +117,7 @@ update-version:
addon/flamenco/__init__.py \
addon/flamenco/manager \
addon/flamenco/manager_README.md \
magefiles/version.go \
web/app/src/manager-api \
web/project-website/data/flamenco.yaml
@echo 'git tag -a -m "Tagged version ${VERSION}" v${VERSION}'
@ -237,28 +142,17 @@ swagger-ui:
@echo
@echo 'Now update pkg/api/static/swagger-ui/index.html to have url: "/api/openapi3.json",'
test:
# Ensure the web-static directory exists, so that `web/web_app.go` can embed something.
mkdir -p ${WEB_STATIC}
go test -short -failfast ./...
test: buildtool
"${BUILDTOOL_PATH}" test
clean:
@go clean -i -x
rm -f flamenco*-v* flamenco-manager flamenco-worker *.exe flamenco-*_race addon-packer stresser
$(MAKE) clean-webapp-static
clean-webapp-static:
# Start with `./` to avoid horrors when WEB_STATIC is absolute (like / or /home/yourname).
rm -rf ./${WEB_STATIC}
# Make sure there is at least something to embed by Go, or it may cause some errors.
mkdir -p ./${WEB_STATIC}
touch ${WEB_STATIC}/emptyfile
clean: buildtool
"${BUILDTOOL_PATH}" clean
devserver-website:
go run ${HUGO_PKG} -s web/project-website serve
devserver-webapp:
yarn --cwd web/app run dev --host
devserver-webapp: buildtool
"${BUILDTOOL_PATH}" devServerWebapp
deploy-website:
$(MAKE) -s check-environment
@ -417,4 +311,4 @@ publish-release-packages:
${RELEASE_PACKAGE_LINUX} ${RELEASE_PACKAGE_DARWIN} ${RELEASE_PACKAGE_DARWIN_ARM64} ${RELEASE_PACKAGE_WINDOWS} ${RELEASE_PACKAGE_SHAFILE} \
${WEBSERVER_SSH}:${WEBSERVER_ROOT}/downloads/
.PHONY: application version flamenco-manager flamenco-worker flamenco-manager_race flamenco-worker_race webapp webapp-static generate generate-go generate-py with-deps swagger-ui list-embedded test clean clean-webapp-static
.PHONY: application version flamenco-manager flamenco-worker flamenco-manager_race flamenco-worker_race webapp webapp-static generate generate-go generate-py with-deps swagger-ui list-embedded test clean flamenco-manager-without-webapp

View File

@ -1,167 +0,0 @@
package main
// SPDX-License-Identifier: GPL-3.0-or-later
import (
"archive/zip"
"compress/flate"
"flag"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
"github.com/mattn/go-colorable"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"projects.blender.org/studio/flamenco/internal/appinfo"
)
var cliArgs struct {
// Do-and-quit flags.
version bool
// Logging level flags.
quiet, debug, trace bool
filename string
}
func main() {
parseCliArgs()
if cliArgs.version {
fmt.Println(appinfo.ApplicationVersion)
return
}
output := zerolog.ConsoleWriter{Out: colorable.NewColorableStdout(), TimeFormat: time.RFC3339}
log.Logger = log.Output(output)
configLogLevel()
outfile, err := filepath.Abs(cliArgs.filename)
if err != nil {
log.Fatal().Err(err).Str("filepath", cliArgs.filename).Msg("unable make output file path absolute")
}
// Open the output file.
logger := log.With().Str("zipname", outfile).Logger()
logger.Info().Msg("creating ZIP file")
zipFile, err := os.Create(outfile)
if err != nil {
logger.Fatal().Err(err).Msg("error creating file")
}
defer zipFile.Close()
zipWriter := zip.NewWriter(zipFile)
zipWriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
return flate.NewWriter(out, flate.BestCompression)
})
// CD to the addon dir to get the relative paths nice.
if err := os.Chdir("addon"); err != nil {
log.Fatal().Err(err).Msg("unable to cd to addon")
}
basePath, err := os.Getwd()
if err != nil {
logger.Fatal().Err(err).Msg("error getting current working directory")
}
// Copy all the files into the ZIP.
addToZip := func(path string, d fs.DirEntry, err error) error {
sublog := log.With().Str("path", path).Logger()
if err != nil {
sublog.Error().Err(err).Msg("error received from filepath.WalkDir, aborting")
return err
}
// Construct the path inside the ZIP file.
relpath, err := filepath.Rel(basePath, path)
if err != nil {
return fmt.Errorf("making %s relative to %s: %w", path, basePath, err)
}
if d.IsDir() {
switch {
case filepath.Base(path) == "__pycache__":
return fs.SkipDir
case relpath == filepath.Join("flamenco", "manager", "docs"):
return fs.SkipDir
case strings.HasPrefix(filepath.Base(path), "."):
// Skip directories like .mypy_cache, etc.
return fs.SkipDir
default:
// Just recurse into this directory.
return nil
}
}
sublog.Debug().Str("path", relpath).Msg("adding file to ZIP")
// Read the file's contents. These are just Python files and maybe a Wheel,
// nothing huge.
contents, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("reading %s: %w", path, err)
}
// Write into the ZIP file.
fileInZip, err := zipWriter.Create(relpath)
if err != nil {
return fmt.Errorf("creating %s in ZIP: %w", relpath, err)
}
_, err = fileInZip.Write(contents)
if err != nil {
return fmt.Errorf("writing to %s in ZIP: %w", relpath, err)
}
return nil
}
logger.Debug().Str("cwd", basePath).Msg("walking directory")
if err := filepath.WalkDir(filepath.Join(basePath, "flamenco"), addToZip); err != nil {
logger.Fatal().Err(err).Msg("error filling ZIP file")
}
comment := fmt.Sprintf("%s add-on for Blender, version %s",
appinfo.ApplicationName,
appinfo.ApplicationVersion,
)
if err := zipWriter.SetComment(comment); err != nil {
logger.Fatal().Err(err).Msg("error setting ZIP comment")
}
if err := zipWriter.Close(); err != nil {
logger.Fatal().Err(err).Msg("error closing ZIP file")
}
}
func parseCliArgs() {
flag.BoolVar(&cliArgs.version, "version", false, "Shows the application version, then exits.")
flag.BoolVar(&cliArgs.quiet, "quiet", false, "Only log warning-level and worse.")
flag.BoolVar(&cliArgs.debug, "debug", false, "Enable debug-level logging.")
flag.BoolVar(&cliArgs.trace, "trace", false, "Enable trace-level logging.")
flag.StringVar(&cliArgs.filename, "filename", "web/static/flamenco-addon.zip", "Filename to save the add-on to.")
flag.Parse()
}
func configLogLevel() {
var logLevel zerolog.Level
switch {
case cliArgs.trace:
logLevel = zerolog.TraceLevel
case cliArgs.debug:
logLevel = zerolog.DebugLevel
case cliArgs.quiet:
logLevel = zerolog.WarnLevel
default:
logLevel = zerolog.InfoLevel
}
zerolog.SetGlobalLevel(logLevel)
}

View File

@ -0,0 +1,88 @@
package main
import (
"fmt"
"go/ast"
"go/format"
"go/parser"
"go/token"
"os"
"path/filepath"
"github.com/rs/zerolog/log"
)
const mageFile = "magefiles/version.go"
// updateMagefiles changes the version number in the Mage files.
// Returns whether the file actually changed.
func updateMagefiles() bool {
logger := log.With().Str("filename", mageFile).Logger()
// Parse the mage file as AST.
fset := token.NewFileSet()
astFile, err := parser.ParseFile(fset, mageFile, nil, parser.SkipObjectResolution|parser.ParseComments)
if err != nil {
logger.Fatal().Err(err).Msgf("could not update mage version file")
return false
}
// Perform replacements on the AST.
replacements := map[string]string{
"version": cliArgs.newVersion,
"releaseCycle": releaseCycle,
}
var (
lastIdent *ast.Ident // Last-seen identifier.
anyFieldChanged bool
)
ast.Inspect(astFile, func(node ast.Node) bool {
switch x := node.(type) {
case *ast.Ident:
lastIdent = x
case *ast.BasicLit:
replacement, ok := replacements[lastIdent.Name]
if ok {
newValue := fmt.Sprintf("%q", replacement)
if x.Value != newValue {
logger.Info().
Str("old", x.Value).
Str("new", newValue).
Msg("updating mage version file")
x.Value = newValue
anyFieldChanged = true
}
}
}
return true
})
// Open a temporary file for writing.
mageDir := filepath.Dir(mageFile)
writer, err := os.CreateTemp(mageDir, filepath.Base(mageFile)+"*.go")
if err != nil {
log.Fatal().Err(err).Msgf("cannot create file in %s", mageDir)
}
defer writer.Close()
// Write the altered AST to the temp file.
if err := format.Node(writer, fset, astFile); err != nil {
log.Fatal().Err(err).Msgf("cannot write updated version of %s to %s", mageFile, writer.Name())
}
// Close the file.
if err := writer.Close(); err != nil {
log.Fatal().Err(err).Msgf("cannot close %s", writer.Name())
}
// Overwrite the original mage file with the temp file.
if err := os.Rename(writer.Name(), mageFile); err != nil {
log.Fatal().Err(err).Msgf("cannot rename %s to %s", writer.Name(), mageFile)
}
return anyFieldChanged
}

View File

@ -6,6 +6,7 @@ import (
"flag"
"fmt"
"os"
"strings"
"time"
"github.com/mattn/go-colorable"
@ -13,13 +14,18 @@ import (
"github.com/rs/zerolog/log"
)
var cliArgs struct {
// Global variables used by the updateXXX() functions.
var (
cliArgs struct {
// Logging level flags.
quiet, debug, trace bool
newVersion string
updateMakefile bool
}
}
releaseCycle string
)
func main() {
parseCliArgs()
@ -29,11 +35,23 @@ func main() {
log.Info().Str("version", cliArgs.newVersion).Msg("updating Flamenco version")
switch {
case strings.Contains(cliArgs.newVersion, "alpha"), strings.Contains(cliArgs.newVersion, "dev"):
releaseCycle = "alpha"
case strings.Contains(cliArgs.newVersion, "beta"):
releaseCycle = "beta"
case strings.Contains(cliArgs.newVersion, "rc"):
releaseCycle = "rc"
default:
releaseCycle = "release"
}
var anyFileWasChanged bool
if cliArgs.updateMakefile {
anyFileWasChanged = updateMakefile() || anyFileWasChanged
}
anyFileWasChanged = updateAddon() || anyFileWasChanged
anyFileWasChanged = updateMagefiles() || anyFileWasChanged
if !anyFileWasChanged {
log.Warn().Msg("nothing changed")

24
go.mod
View File

@ -19,22 +19,27 @@ require (
github.com/google/uuid v1.5.0
github.com/graarh/golang-socketio v0.0.0-20170510162725-2c44953b9b5f
github.com/labstack/echo/v4 v4.9.1
github.com/magefile/mage v1.15.0
github.com/mattn/go-colorable v0.1.12
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
github.com/pressly/goose/v3 v3.15.1
github.com/rs/zerolog v1.26.1
github.com/stretchr/testify v1.8.4
github.com/zcalusic/sysinfo v1.0.1
github.com/ziflex/lecho/v3 v3.1.0
golang.org/x/crypto v0.23.0
golang.org/x/crypto v0.25.0
golang.org/x/image v0.18.0
golang.org/x/net v0.25.0
golang.org/x/sys v0.20.0
golang.org/x/net v0.27.0
golang.org/x/sync v0.7.0
golang.org/x/sys v0.22.0
golang.org/x/vuln v1.1.3
gopkg.in/yaml.v2 v2.4.0
honnef.co/go/tools v0.5.1
modernc.org/sqlite v1.28.0
)
require (
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.7.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
@ -54,11 +59,12 @@ require (
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.1 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
golang.org/x/mod v0.19.0 // indirect
golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.org/x/tools v0.23.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/uint128 v1.3.0 // indirect
modernc.org/cc/v3 v3.41.0 // indirect
@ -70,3 +76,7 @@ require (
modernc.org/strutil v1.2.0 // indirect
modernc.org/token v1.1.0 // indirect
)
// Replace staticcheck release with a specific revision of their `main` branch,
// so that it includes my PR https://github.com/dominikh/go-tools/pull/1597
replace honnef.co/go/tools v0.5.1 => honnef.co/go/tools v0.0.0-20240920144234-9f4b51e3ab5a

38
go.sum generated
View File

@ -1,3 +1,5 @@
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0=
@ -115,6 +117,8 @@ github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++
github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc=
github.com/lestrrat-go/jwx v1.2.7/go.mod h1:bw24IXWbavc0R2RsOtpXL7RtMyP589yZ1+L7kd09ZGA=
github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
@ -138,8 +142,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@ -191,8 +195,10 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ=
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
@ -200,8 +206,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@ -211,8 +217,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -235,7 +241,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -247,9 +252,12 @@ golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7 h1:FemxDzfMUcK2f3YY4H+05K9CDzbSVr2+q/JKN45pey0=
golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@ -271,8 +279,10 @@ golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
golang.org/x/vuln v1.1.3 h1:NPGnvPOTgnjBc9HTaUx+nj+EaUYxl5SJOWqaDYGaFYw=
golang.org/x/vuln v1.1.3/go.mod h1:7Le6Fadm5FOqE9C926BCD0g12NWyhg7cxV4BwcPFuNY=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -294,6 +304,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20240920144234-9f4b51e3ab5a h1:P0jdAtRz/UFVIXrgf9EdWApQJnbQrmsUwyARz4FF+D8=
honnef.co/go/tools v0.0.0-20240920144234-9f4b51e3ab5a/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs=
lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo=
lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q=

11
mage.go Normal file
View File

@ -0,0 +1,11 @@
//go:build ignore
package main
import (
"os"
"github.com/magefile/mage/mage"
)
func main() { os.Exit(mage.Main()) }

108
magefiles/addonpacker.go Normal file
View File

@ -0,0 +1,108 @@
//go:build mage
package main
// SPDX-License-Identifier: GPL-3.0-or-later
import (
"archive/zip"
"compress/flate"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"projects.blender.org/studio/flamenco/internal/appinfo"
)
func packAddon(filename string) error {
outfile, err := filepath.Abs(filename)
if err != nil {
return fmt.Errorf("unable make output file path absolute: %w", err)
}
// Open the output file.
zipFile, err := os.Create(outfile)
if err != nil {
return fmt.Errorf("error creating file %s: %w", outfile, err)
}
defer zipFile.Close()
zipWriter := zip.NewWriter(zipFile)
defer zipWriter.Close()
zipWriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
return flate.NewWriter(out, flate.BestCompression)
})
basePath, err := filepath.Abs("./addon") // os.Getwd()
if err != nil {
return fmt.Errorf("error getting current working directory: %w", err)
}
// Copy all the files into the ZIP.
addToZip := func(path string, d fs.DirEntry, err error) error {
if err != nil {
return fmt.Errorf("error received from filepath.WalkDir: %w", err)
}
// Construct the path inside the ZIP file.
relpath, err := filepath.Rel(basePath, path)
if err != nil {
return fmt.Errorf("making %s relative to %s: %w", path, basePath, err)
}
if d.IsDir() {
switch {
case filepath.Base(path) == "__pycache__":
return fs.SkipDir
case relpath == filepath.Join("flamenco", "manager", "docs"):
return fs.SkipDir
case strings.HasPrefix(filepath.Base(path), "."):
// Skip directories like .mypy_cache, etc.
return fs.SkipDir
default:
// Just recurse into this directory.
return nil
}
}
// Read the file's contents. These are just Python files and maybe a Wheel,
// nothing huge.
contents, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("reading %s: %w", path, err)
}
// Write into the ZIP file.
fileInZip, err := zipWriter.Create(relpath)
if err != nil {
return fmt.Errorf("creating %s in ZIP: %w", relpath, err)
}
_, err = fileInZip.Write(contents)
if err != nil {
return fmt.Errorf("writing to %s in ZIP: %w", relpath, err)
}
return nil
}
if err := filepath.WalkDir(filepath.Join(basePath, "flamenco"), addToZip); err != nil {
return fmt.Errorf("error filling ZIP file: %w", err)
}
comment := fmt.Sprintf("%s add-on for Blender, version %s",
appinfo.ApplicationName,
appinfo.ApplicationVersion,
)
if err := zipWriter.SetComment(comment); err != nil {
return fmt.Errorf("error setting ZIP comment: %w", err)
}
if err := zipWriter.Close(); err != nil {
return fmt.Errorf("error closing ZIP file: %w", err)
}
return nil
}

134
magefiles/build.go Normal file
View File

@ -0,0 +1,134 @@
//go:build mage
package main
// SPDX-License-Identifier: GPL-3.0-or-later
import (
"fmt"
"path/filepath"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/magefile/mage/target"
)
const (
goPkg = "projects.blender.org/studio/flamenco"
)
var (
// The directory that will contain the built webapp files, and some other
// files that will be served as static files by the Flamenco Manager web
// server.
webStatic = filepath.Join("web", "static")
)
// Build Flamenco Manager and Flamenco Worker, including the webapp and the add-on
func Build() {
mg.Deps(FlamencoManager, FlamencoWorker)
}
// Build Flamenco Manager with the webapp and add-on ZIP embedded
func FlamencoManager() error {
mg.Deps(WebappStatic)
mg.Deps(flamencoManager)
return nil
}
// Only build the Flamenco Manager executable, do not rebuild the webapp
func FlamencoManagerWithoutWebapp() error {
mg.Deps(flamencoManager)
return nil
}
func flamencoManager() error {
return build("./cmd/flamenco-manager")
}
// Build the Flamenco Worker executable
func FlamencoWorker() error {
return build("./cmd/flamenco-worker")
}
// Build the webapp as static files that can be served
func WebappStatic() error {
runInstall, err := target.Dir("web/app/node_modules")
if err != nil {
return err
}
if runInstall {
mg.SerialDeps(WebappInstallDeps)
}
if err := cleanWebappStatic(); err != nil {
return err
}
env := map[string]string{
"MSYS2_ARG_CONV_EXCL": "*",
}
// When changing the base URL, also update the line
// e.GET("/app/*", echo.WrapHandler(webAppHandler))
// in `cmd/flamenco-manager/main.go`
err = sh.RunWithV(env,
"yarn",
"--cwd", "web/app",
"build",
"--outDir", "../static",
"--base=/app/",
"--logLevel", "warn",
// For debugging you can add:
// "--minify", "false",
)
if err != nil {
return err
}
fmt.Printf("Web app has been installed into %s\n", webStatic)
// Build the add-on ZIP as it's part of the static web files.
zipPath := filepath.Join(webStatic, "flamenco3-addon.zip")
return packAddon(zipPath)
}
// Use Yarn to install the webapp's NodeJS dependencies
func WebappInstallDeps() error {
env := map[string]string{
"MSYS2_ARG_CONV_EXCL": "*",
}
return sh.RunWithV(env,
"yarn",
"--cwd", "web/app",
"install",
)
}
func build(exePackage string) error {
flags, err := buildFlags()
if err != nil {
return err
}
args := []string{"build", "-v"}
args = append(args, flags...)
args = append(args, exePackage)
return sh.RunV(mg.GoCmd(), args...)
}
func buildFlags() ([]string, error) {
hash, err := gitHash()
if err != nil {
return nil, err
}
ldflags := "" +
fmt.Sprintf(" -X %s/internal/appinfo.ApplicationVersion=%s", goPkg, version) +
fmt.Sprintf(" -X %s/internal/appinfo.ApplicationGitHash=%s", goPkg, hash) +
fmt.Sprintf(" -X %s/internal/appinfo.ReleaseCycle=%s", goPkg, releaseCycle)
flags := []string{
"-ldflags=" + ldflags,
}
return flags, nil
}

61
magefiles/check.go Normal file
View File

@ -0,0 +1,61 @@
//go:build mage
package main
import (
"context"
"errors"
"fmt"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"golang.org/x/vuln/scan"
"honnef.co/go/tools/lintcmd"
lintcmdversion "honnef.co/go/tools/lintcmd/version"
"honnef.co/go/tools/simple"
"honnef.co/go/tools/staticcheck"
"honnef.co/go/tools/stylecheck"
"honnef.co/go/tools/unused"
)
// Run unit tests, check for vulnerabilities, and run the linter
func Check(ctx context.Context) {
mg.CtxDeps(ctx, Test, Govulncheck, Staticcheck, Vet)
}
// Run unit tests
func Test(ctx context.Context) error {
return sh.RunV(mg.GoCmd(), "test", "-short", "-failfast", "./...")
}
// Check for known vulnerabilities.
func Govulncheck(ctx context.Context) error {
cmd := scan.Command(ctx, "./...")
if err := cmd.Start(); err != nil {
return err
}
return cmd.Wait()
}
// Analyse the source code.
func Staticcheck() error {
cmd := lintcmd.NewCommand("staticcheck")
cmd.SetVersion(lintcmdversion.Version, lintcmdversion.MachineVersion)
cmd.ParseFlags([]string{"./..."})
cmd.AddAnalyzers(simple.Analyzers...)
cmd.AddAnalyzers(staticcheck.Analyzers...)
cmd.AddAnalyzers(stylecheck.Analyzers...)
cmd.AddAnalyzers(unused.Analyzer)
exitCode := cmd.Execute()
if exitCode != 0 {
return errors.New("staticcheck failed")
}
fmt.Println("staticcheck ok")
return nil
}
// Run `go vet`
func Vet() error {
return sh.RunV(mg.GoCmd(), "vet", "./...")
}

64
magefiles/clean.go Normal file
View File

@ -0,0 +1,64 @@
//go:build mage
package main
import (
"fmt"
"os"
"path/filepath"
"github.com/magefile/mage/sh"
)
// Remove executables and other build output
func Clean() error {
if err := cleanWebappStatic(); err != nil {
return err
}
if err := sh.Run("go", "clean"); err != nil {
return err
}
if err := rm(
"flamenco-manager", "flamenco-manager.exe",
"flamenco-manager_race", "flamenco-manager_race.exe",
"flamenco-worker", "flamenco-worker.exe",
"flamenco-worker_race", "flamenco-worker_race.exe",
); err != nil {
return err
}
return nil
}
func cleanWebappStatic() error {
// Just a simple heuristic to avoid deleting things like "/" or "C:\"
if len(webStatic) < 4 {
panic(fmt.Sprintf("webStatic path is too short, I don't trust it: %q", webStatic))
}
if err := sh.Rm(webStatic); err != nil {
return fmt.Errorf("unable to remove old web static dir %q: %w", webStatic, err)
}
if err := os.MkdirAll(webStatic, os.ModePerm); err != nil {
return fmt.Errorf("unable to create web static dir %q: %w", webStatic, err)
}
// Make sure there is at least something to embed by Go, or it may cause some
// errors. This is done in the 'clean' function so that the Go code can be
// built before building the webapp.
emptyfile := filepath.Join(webStatic, "emptyfile")
if err := os.WriteFile(emptyfile, []byte{}, os.ModePerm); err != nil {
return err
}
return nil
}
func rm(path ...string) error {
for _, p := range path {
if err := sh.Rm(p); err != nil {
return err
}
}
return nil
}

13
magefiles/devserver.go Normal file
View File

@ -0,0 +1,13 @@
//go:build mage
package main
// SPDX-License-Identifier: GPL-3.0-or-later
import (
"github.com/magefile/mage/sh"
)
func DevServerWebapp() error {
return sh.RunV("yarn", "--cwd", "web/app", "run", "dev", "--host")
}

172
magefiles/generate.go Normal file
View File

@ -0,0 +1,172 @@
//go:build mage
package main
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
)
// Generate code (OpenAPI and test mocks)
func Generate() {
mg.Deps(GenerateGo, GeneratePy, GenerateJS)
}
// Generate Go code for Flamenco Manager and Worker
func GenerateGo(ctx context.Context) error {
r := NewRunner(ctx)
r.Run(mg.GoCmd(), "generate", "./pkg/api/...")
r.Run(mg.GoCmd(), "generate", "./internal/...")
if err := r.Wait(); err != nil {
return err
}
// The generators always produce UNIX line-ends. This creates false file
// modifications with Git. Convert them to DOS line-ends to avoid this.
if runtime.GOOS == "windows" {
unix2dosModifiedFiles(".gen.go$")
}
return nil
}
// Generate Python code for the add-on
func GeneratePy() error {
// The generator doesn't consistently overwrite existing files, nor does it
// remove no-longer-generated files.
sh.Rm("addon/flamenco/manager")
// See https://openapi-generator.tech/docs/generators/python for the options.
err := sh.Run("java",
"-jar", "addon/openapi-generator-cli.jar",
"generate",
"-i", "pkg/api/flamenco-openapi.yaml",
"-g", "python",
"-o", "addon/",
"--package-name", "flamenco.manager",
"--http-user-agent", fmt.Sprintf("Flamenco/%s (Blender add-on)", version),
"-p", "generateSourceCodeOnly=true",
"-p", "projectName=Flamenco",
"-p", fmt.Sprintf("packageVersion=%s", version))
if err != nil {
return err
}
// The generator outputs files so that we can write our own tests. We don't,
// though, so it's better to just remove those placeholders.
sh.Rm("addon/flamenco/manager/test")
// The generators always produce UNIX line-ends. This creates false file
// modifications with Git. Convert them to DOS line-ends to avoid this.
if runtime.GOOS == "windows" {
unix2dosModifiedFiles("addon/flamenco/manager")
}
return nil
}
// Generate JavaScript code for the webapp
func GenerateJS() error {
const (
jsOutDir = "web/app/src/manager-api"
jsTempDir = "web/_tmp-manager-api-javascript"
)
sh.Rm(jsOutDir)
sh.Rm(jsTempDir)
// See https://openapi-generator.tech/docs/generators/javascript for the options.
// Version '0.0.0' is used as NPM doesn't like Git hashes as versions.
//
// -p modelPropertyNaming=original is needed because otherwise the generator will
// use original naming internally, but generate docs with camelCase, and then
// things don't work properly.
err := sh.Run("java",
"-jar", "addon/openapi-generator-cli.jar",
"generate",
"-i", "pkg/api/flamenco-openapi.yaml",
"-g", "javascript",
"-o", jsTempDir,
"--http-user-agent", fmt.Sprintf("Flamenco/%s / webbrowser", version),
"-p", "projectName=flamenco-manager",
"-p", "projectVersion=0.0.0",
"-p", "apiPackage=manager",
"-p", "disallowAdditionalPropertiesIfNotPresent=false",
"-p", "usePromises=true",
"-p", "moduleName=flamencoManager")
if err != nil {
return err
}
// Cherry-pick the generated sources, and remove everything else.
if err := os.Rename(filepath.Join(jsTempDir, "src"), jsOutDir); err != nil {
return err
}
sh.Rm(jsTempDir)
if runtime.GOOS == "windows" {
unix2dosModifiedFiles(jsOutDir)
}
return nil
}
// unix2dosModifiedFiles changes line ends in files Git considers modified that match the given pattern.
func unix2dosModifiedFiles(pattern string) {
// Get modified files from Git. Expected lines like:
//
// M pkg/api/openapi_client.gen.go
// M pkg/api/openapi_server.gen.go
// M pkg/api/openapi_spec.gen.go
// M pkg/api/openapi_types.gen.go
// ?? magefiles/generate.go
gitStatus, err := sh.Output("git", "status", "--porcelain")
if err != nil {
panic(fmt.Sprintf("error running 'git status': %s", err))
}
// Construct a list of modified files that match the pattern.
patternRe := regexp.MustCompile(pattern)
modified := []string{}
for _, line := range strings.Split(gitStatus, "\n") {
if line[0:3] != " M " {
continue
}
if !patternRe.MatchString(line[3:]) {
continue
}
modified = append(modified, line[3:])
}
// Run unix2dos on all found files.
for _, path := range modified {
unix2dos(path)
}
}
func unix2dos(filename string) {
if mg.Verbose() {
fmt.Printf("unix2dos %s\n", filename)
}
// TODO: rewrite to stream the data instead of reading, copying, and writing
// everything.
contents, err := os.ReadFile(filename)
if err != nil {
panic(fmt.Sprintf("error converting UNIX to DOS line ends: %v", err))
}
lines := bytes.Split(contents, []byte("\n"))
err = os.WriteFile(filename, bytes.Join(lines, []byte("\r\n")), os.ModePerm)
if err != nil {
panic(fmt.Sprintf("error writing DOS line ends: %v", err))
}
}

45
magefiles/install.go Normal file
View File

@ -0,0 +1,45 @@
//go:build mage
package main
// SPDX-License-Identifier: GPL-3.0-or-later
import (
"context"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
)
var (
generators = []string{
"github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.9.0",
"github.com/golang/mock/mockgen@v1.6.0",
}
)
// Install build-time dependencies: code generators and NodeJS dependencies.
func InstallDeps() {
mg.SerialDeps(InstallGenerators, InstallDepsWebapp)
}
// Install code generators.
func InstallGenerators(ctx context.Context) error {
r := NewRunner(ctx)
for _, pkg := range generators {
r.Run(mg.GoCmd(), "install", pkg)
}
return r.Wait()
}
// Use Yarn to install the webapp's NodeJS dependencies
func InstallDepsWebapp() error {
env := map[string]string{
"MSYS2_ARG_CONV_EXCL": "*",
}
return sh.RunWithV(env,
"yarn",
"--cwd", "web/app",
"install",
)
}

48
magefiles/runner.go Normal file
View File

@ -0,0 +1,48 @@
//go:build mage
package main
import (
"context"
"github.com/magefile/mage/sh"
"golang.org/x/sync/errgroup"
)
// Runner allows running a group of commands sequentially, stopping at the first
// failure.
// See https://github.com/magefile/mage/issues/455 for the feature request
// to include this in Mage.
type Runner struct {
group *errgroup.Group
ctx context.Context
}
// NewRunner constructs a new runner that's bound to the given context. If the
// context is done, no new command will be executed. It does NOT abort an
// already-running command.
func NewRunner(ctx context.Context) *Runner {
group, groupctx := errgroup.WithContext(ctx)
group.SetLimit(1)
return &Runner{
group: group,
ctx: groupctx,
}
}
// Run the given command.
// This only runs a command if no previous command has failed yet.
func (r *Runner) Run(cmd string, args ...string) {
r.group.Go(func() error {
if err := r.ctx.Err(); err != nil {
return err
}
return sh.RunV(cmd, args...)
})
}
// Wait for the commands to finish running, and return any error.
func (r *Runner) Wait() error {
return r.group.Wait()
}

33
magefiles/version.go Normal file
View File

@ -0,0 +1,33 @@
//go:build mage
package main
import (
"fmt"
"github.com/magefile/mage/sh"
)
// To update the version number in all the relevant places, update the VERSION
// variable below and run `mage update-version`.
const (
version = "3.6-alpha5"
releaseCycle = "alpha"
)
func gitHash() (string, error) {
return sh.Output("git", "rev-parse", "--short", "HEAD")
}
// Show which version information would be embedded in executables
func Version() error {
fmt.Printf("Package : %s\n", goPkg)
fmt.Printf("Version : %s\n", version)
hash, err := gitHash()
if err != nil {
return err
}
fmt.Printf("Git Hash : %s\n", hash)
return nil
}