//go:build mage package main // SPDX-License-Identifier: GPL-3.0-or-later import ( "context" "fmt" "os" "path/filepath" "runtime" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" "github.com/magefile/mage/target" ) const ( goPkg = "projects.blender.org/studio/flamenco" ) var ( // The directory that will contain the built webapp files, and some other // files that will be served as static files by the Flamenco Manager web // server. webStatic = filepath.Join("web", "static") ) // Build Flamenco Manager and Flamenco Worker, including the webapp and the add-on func Build() { mg.Deps(FlamencoManager, FlamencoWorker) } // BuildOptimized uses caching and parallelization for faster builds func BuildOptimized() error { return BuildOptimizedWithContext(context.Background()) } // BuildOptimizedWithContext builds with caching and parallelization func BuildOptimizedWithContext(ctx context.Context) error { cache := NewBuildCache() // Warm cache and check what needs building if err := WarmBuildCache(cache); err != nil { fmt.Printf("Warning: Failed to warm build cache: %v\n", err) } // Define build tasks with dependencies tasks := []*BuildTask{ CreateGenerateTask("generate-go", []string{}, func() error { return buildOptimizedGenerateGo(cache) }), CreateGenerateTask("generate-py", []string{}, func() error { return buildOptimizedGeneratePy(cache) }), CreateGenerateTask("generate-js", []string{}, func() error { return buildOptimizedGenerateJS(cache) }), CreateWebappTask("webapp-static", []string{"generate-js"}, func() error { return buildOptimizedWebappStatic(cache) }), CreateBuildTask("manager", []string{"generate-go", "webapp-static"}, func() error { return buildOptimizedManager(cache) }), CreateBuildTask("worker", []string{"generate-go"}, func() error { return buildOptimizedWorker(cache) }), } // Determine optimal concurrency maxConcurrency := runtime.NumCPU() if maxConcurrency > 4 { maxConcurrency = 4 // Reasonable limit for build tasks } builder := NewParallelBuilder(maxConcurrency) return builder.ExecuteParallel(ctx, tasks) } // BuildIncremental performs incremental build with caching func BuildIncremental() error { cache := NewBuildCache() fmt.Println("Build: Starting incremental build with caching") // Check and build each component incrementally if err := buildIncrementalGenerate(cache); err != nil { return err } if err := buildIncrementalWebapp(cache); err != nil { return err } if err := buildIncrementalBinaries(cache); err != nil { return err } fmt.Println("Build: Incremental build completed successfully") return nil } // Build Flamenco Manager with the webapp and add-on ZIP embedded func FlamencoManager() error { mg.Deps(WebappStatic) mg.Deps(flamencoManager) return nil } // Only build the Flamenco Manager executable, do not rebuild the webapp func FlamencoManagerWithoutWebapp() error { mg.Deps(flamencoManager) return nil } // Build the Flamenco Manager executable with race condition checker enabled, do not rebuild the webapp func FlamencoManagerRace() error { return build("./cmd/flamenco-manager", "-race") } func flamencoManager() error { return build("./cmd/flamenco-manager") } // Build the Flamenco Worker executable func FlamencoWorker() error { return build("./cmd/flamenco-worker") } // Build the webapp as static files that can be served func WebappStatic() error { runInstall, err := target.Dir("web/app/node_modules") if err != nil { return err } if runInstall { mg.SerialDeps(InstallDepsWebapp) } if err := cleanWebappStatic(); err != nil { return err } env := map[string]string{ "MSYS2_ARG_CONV_EXCL": "*", } // When changing the base URL, also update the line // e.GET("/app/*", echo.WrapHandler(webAppHandler)) // in `cmd/flamenco-manager/main.go` err = sh.RunWithV(env, "yarn", "--cwd", "web/app", "build", "--outDir", "../static", "--base=/app/", "--logLevel", "warn", // For debugging you can add: // "--minify", "false", ) if err != nil { return err } fmt.Printf("Web app has been installed into %s\n", webStatic) // Build the add-on ZIP as it's part of the static web files. zipPath := filepath.Join(webStatic, "flamenco-addon.zip") return packAddon(zipPath) } func build(exePackage string, extraArgs ...string) error { flags, err := buildFlags() if err != nil { return err } args := []string{"build", "-v"} args = append(args, flags...) args = append(args, extraArgs...) args = append(args, exePackage) return sh.RunV(mg.GoCmd(), args...) } func buildFlags() ([]string, error) { hash, err := gitHash() if err != nil { return nil, err } ldflags := os.Getenv("LDFLAGS") + fmt.Sprintf(" -X %s/internal/appinfo.ApplicationVersion=%s", goPkg, version) + fmt.Sprintf(" -X %s/internal/appinfo.ApplicationGitHash=%s", goPkg, hash) + fmt.Sprintf(" -X %s/internal/appinfo.ReleaseCycle=%s", goPkg, releaseCycle) flags := []string{ "-ldflags=" + ldflags, } return flags, nil } // Optimized build functions with caching // buildOptimizedGenerateGo generates Go code with caching func buildOptimizedGenerateGo(cache *BuildCache) error { sources := []string{ "pkg/api/flamenco-openapi.yaml", "pkg/api/*.gen.go", "internal/**/*.go", } outputs := []string{ "pkg/api/openapi_client.gen.go", "pkg/api/openapi_server.gen.go", "pkg/api/openapi_spec.gen.go", "pkg/api/openapi_types.gen.go", } needsBuild, err := cache.NeedsBuild("generate-go", sources, []string{}, outputs) if err != nil { return err } if !needsBuild { fmt.Println("Cache: Go code generation is up to date") return cache.RestoreFromCache("generate-go", outputs) } fmt.Println("Cache: Generating Go code") if err := GenerateGo(context.Background()); err != nil { return err } // Record successful build and cache artifacts if err := cache.RecordBuild("generate-go", sources, []string{}, outputs); err != nil { return err } return cache.CopyToCache("generate-go", outputs) } // buildOptimizedGeneratePy generates Python code with caching func buildOptimizedGeneratePy(cache *BuildCache) error { sources := []string{ "pkg/api/flamenco-openapi.yaml", "addon/openapi-generator-cli.jar", } outputs := []string{ "addon/flamenco/manager/", } needsBuild, err := cache.NeedsBuild("generate-py", sources, []string{}, outputs) if err != nil { return err } if !needsBuild { fmt.Println("Cache: Python code generation is up to date") return nil // Directory outputs are harder to cache/restore } fmt.Println("Cache: Generating Python code") return GeneratePy() } // buildOptimizedGenerateJS generates JavaScript code with caching func buildOptimizedGenerateJS(cache *BuildCache) error { sources := []string{ "pkg/api/flamenco-openapi.yaml", "addon/openapi-generator-cli.jar", } outputs := []string{ "web/app/src/manager-api/", } needsBuild, err := cache.NeedsBuild("generate-js", sources, []string{}, outputs) if err != nil { return err } if !needsBuild { fmt.Println("Cache: JavaScript code generation is up to date") return nil // Directory outputs are harder to cache/restore } fmt.Println("Cache: Generating JavaScript code") return GenerateJS() } // buildOptimizedWebappStatic builds webapp with caching func buildOptimizedWebappStatic(cache *BuildCache) error { sources := []string{ "web/app/**/*.ts", "web/app/**/*.vue", "web/app/**/*.js", "web/app/package.json", "web/app/yarn.lock", "web/app/src/manager-api/**/*.js", } needsBuild, err := cache.NeedsBuild("webapp-static", sources, []string{"generate-js"}, []string{webStatic}) if err != nil { return err } if !needsBuild { fmt.Println("Cache: Webapp static files are up to date") return nil // Static directory is the output } fmt.Println("Cache: Building webapp static files") if err := WebappStatic(); err != nil { return err } // Record successful build return cache.RecordBuild("webapp-static", sources, []string{"generate-js"}, []string{webStatic}) } // buildOptimizedManager builds manager binary with caching func buildOptimizedManager(cache *BuildCache) error { sources := []string{ "cmd/flamenco-manager/**/*.go", "internal/manager/**/*.go", "pkg/**/*.go", "go.mod", "go.sum", } outputs := []string{ "flamenco-manager", "flamenco-manager.exe", } needsBuild, err := cache.NeedsBuild("manager", sources, []string{"generate-go", "webapp-static"}, outputs) if err != nil { return err } if !needsBuild { fmt.Println("Cache: Manager binary is up to date") return cache.RestoreFromCache("manager", outputs) } fmt.Println("Cache: Building manager binary") if err := build("./cmd/flamenco-manager"); err != nil { return err } // Record successful build and cache binary if err := cache.RecordBuild("manager", sources, []string{"generate-go", "webapp-static"}, outputs); err != nil { return err } return cache.CopyToCache("manager", outputs) } // buildOptimizedWorker builds worker binary with caching func buildOptimizedWorker(cache *BuildCache) error { sources := []string{ "cmd/flamenco-worker/**/*.go", "internal/worker/**/*.go", "pkg/**/*.go", "go.mod", "go.sum", } outputs := []string{ "flamenco-worker", "flamenco-worker.exe", } needsBuild, err := cache.NeedsBuild("worker", sources, []string{"generate-go"}, outputs) if err != nil { return err } if !needsBuild { fmt.Println("Cache: Worker binary is up to date") return cache.RestoreFromCache("worker", outputs) } fmt.Println("Cache: Building worker binary") if err := build("./cmd/flamenco-worker"); err != nil { return err } // Record successful build and cache binary if err := cache.RecordBuild("worker", sources, []string{"generate-go"}, outputs); err != nil { return err } return cache.CopyToCache("worker", outputs) } // Incremental build functions // buildIncrementalGenerate handles incremental code generation func buildIncrementalGenerate(cache *BuildCache) error { fmt.Println("Build: Checking code generation") // Check each generation step independently tasks := []struct { name string fn func() error }{ {"Go generation", func() error { return buildOptimizedGenerateGo(cache) }}, {"Python generation", func() error { return buildOptimizedGeneratePy(cache) }}, {"JavaScript generation", func() error { return buildOptimizedGenerateJS(cache) }}, } for _, task := range tasks { if err := task.fn(); err != nil { return fmt.Errorf("%s failed: %w", task.name, err) } } return nil } // buildIncrementalWebapp handles incremental webapp building func buildIncrementalWebapp(cache *BuildCache) error { fmt.Println("Build: Checking webapp") return buildOptimizedWebappStatic(cache) } // buildIncrementalBinaries handles incremental binary building func buildIncrementalBinaries(cache *BuildCache) error { fmt.Println("Build: Checking binaries") // Check manager if err := buildOptimizedManager(cache); err != nil { return fmt.Errorf("manager build failed: %w", err) } // Check worker if err := buildOptimizedWorker(cache); err != nil { return fmt.Errorf("worker build failed: %w", err) } return nil } // Cache management functions // CleanCache removes all build cache data func CleanCache() error { cache := NewBuildCache() return cache.CleanCache() } // CacheStatus shows build cache statistics func CacheStatus() error { cache := NewBuildCache() stats, err := cache.CacheStats() if err != nil { return err } fmt.Println("Build Cache Status:") fmt.Printf(" Targets cached: %d\n", stats["targets_cached"]) fmt.Printf(" Cache size: %d MB (%d bytes)\n", stats["cache_size_mb"], stats["cache_size_bytes"]) return nil }