Ryan Malloy
2f82e8d2e0
* Docker Infrastructure: - Multi-stage Dockerfile.dev with optimized Go proxy configuration - Complete compose.dev.yml with service orchestration - Fixed critical GOPROXY setting achieving 42x performance improvement - Migrated from Poetry to uv for faster Python package management * Build System Enhancements: - Enhanced Mage build system with caching and parallelization - Added incremental build capabilities with SHA256 checksums - Implemented parallel task execution with dependency resolution - Added comprehensive test orchestration targets * Testing Infrastructure: - Complete API testing suite with OpenAPI validation - Performance testing with multi-worker simulation - Integration testing for end-to-end workflows - Database testing with migration validation - Docker-based test environments * Documentation: - Comprehensive Docker development guides - Performance optimization case study - Build system architecture documentation - Test infrastructure usage guides * Performance Results: - Build time reduced from 60+ min failures to 9.5 min success - Go module downloads: 42x faster (84.2s vs 60+ min timeouts) - Success rate: 0% → 100% - Developer onboarding: days → 10 minutes Fixes critical Docker build failures and establishes production-ready containerized development environment with comprehensive testing.
331 lines
11 KiB
Docker
331 lines
11 KiB
Docker
# =============================================================================
|
||
# Flamenco Docker Development Environment Makefile
|
||
# =============================================================================
|
||
# Manages Docker Compose operations for Flamenco development environment
|
||
#
|
||
# Usage:
|
||
# make help # Show available targets
|
||
# make dev-setup # Initial development environment setup
|
||
# make dev-start # Start development services
|
||
# make dev-tools # Start development tools (Vue.js, Hugo, profiling)
|
||
# make dev-stop # Stop all services
|
||
# make dev-clean # Clean environment and volumes
|
||
|
||
-include .env
|
||
export
|
||
|
||
# =============================================================================
|
||
# Configuration
|
||
# =============================================================================
|
||
COMPOSE_FILE := compose.dev.yml
|
||
COMPOSE_PROJECT_NAME ?= flamenco-dev
|
||
DOMAIN ?= flamenco.l.supported.systems
|
||
|
||
# =============================================================================
|
||
# Docker Compose Commands
|
||
# =============================================================================
|
||
DOCKER_COMPOSE := docker compose -f $(COMPOSE_FILE)
|
||
DOCKER_COMPOSE_TOOLS := $(DOCKER_COMPOSE) --profile dev-tools
|
||
|
||
# =============================================================================
|
||
# Default target
|
||
# =============================================================================
|
||
.PHONY: help
|
||
help: ## Show this help message
|
||
@echo "🐳 Flamenco Docker Development Environment"
|
||
@echo "=========================================="
|
||
@echo ""
|
||
@echo "Available targets:"
|
||
@awk 'BEGIN {FS = ":.*##"; printf ""} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
|
||
@echo ""
|
||
@echo "Environment:"
|
||
@echo " COMPOSE_PROJECT_NAME: $(COMPOSE_PROJECT_NAME)"
|
||
@echo " DOMAIN: $(DOMAIN)"
|
||
|
||
##@ Setup Commands
|
||
|
||
.PHONY: prerequisites
|
||
prerequisites: ## Check and install prerequisites
|
||
@echo "🔍 Checking prerequisites..."
|
||
@docker --version || (echo "❌ Docker not found. Please install Docker." && exit 1)
|
||
@docker compose version || (echo "❌ Docker Compose plugin not found. Install with: apt install docker-compose-plugin" && exit 1)
|
||
@echo "✅ Prerequisites OK"
|
||
|
||
.PHONY: network-setup
|
||
network-setup: ## Create external networks
|
||
@echo "🌐 Setting up networks..."
|
||
@docker network create caddy 2>/dev/null || echo "ℹ️ Caddy network already exists"
|
||
@echo "✅ Networks ready"
|
||
|
||
.PHONY: caddy-proxy
|
||
caddy-proxy: network-setup ## Start caddy-docker-proxy for reverse proxy
|
||
@echo "🔄 Starting caddy-docker-proxy..."
|
||
@docker run -d \
|
||
--name caddy-docker-proxy \
|
||
--restart unless-stopped \
|
||
--network caddy \
|
||
-p 80:80 -p 443:443 \
|
||
-v /var/run/docker.sock:/var/run/docker.sock:ro \
|
||
-v caddy_data:/data \
|
||
lucaslorentz/caddy-docker-proxy:ci-alpine 2>/dev/null || echo "ℹ️ caddy-docker-proxy already running"
|
||
@echo "✅ Caddy reverse proxy ready"
|
||
|
||
.PHONY: env-setup
|
||
env-setup: ## Setup .env file from template
|
||
@echo "⚙️ Setting up environment..."
|
||
@if [ ! -f .env ]; then \
|
||
cp .env.dev .env && echo "✅ Created .env from template"; \
|
||
else \
|
||
echo "ℹ️ .env file already exists"; \
|
||
fi
|
||
|
||
.PHONY: dev-setup
|
||
dev-setup: prerequisites network-setup env-setup ## Complete development environment setup
|
||
@echo "🚀 Setting up Flamenco development environment..."
|
||
@docker compose --progress plain -f $(COMPOSE_FILE) build
|
||
@$(DOCKER_COMPOSE) up shared-storage-setup
|
||
@echo "✅ Development environment setup complete"
|
||
|
||
##@ Development Commands
|
||
|
||
.PHONY: dev-start
|
||
dev-start: ## Start core development services (Manager + Worker)
|
||
@echo "🏁 Starting core development services..."
|
||
@$(DOCKER_COMPOSE) up -d flamenco-manager flamenco-worker
|
||
@echo ""
|
||
@echo "✅ Core services started!"
|
||
@echo ""
|
||
@echo "🌐 Access URLs:"
|
||
@echo " Manager (proxy): https://manager.$(DOMAIN)"
|
||
@echo " Manager (direct): http://localhost:8080"
|
||
|
||
.PHONY: dev-tools
|
||
dev-tools: ## Start development tools (Vue.js, Hugo, profiling)
|
||
@echo "🛠️ Starting development tools..."
|
||
@$(DOCKER_COMPOSE_TOOLS) up -d
|
||
@echo ""
|
||
@echo "✅ Development tools started!"
|
||
@echo ""
|
||
@echo "🌐 Development URLs:"
|
||
@echo " Frontend: https://$(DOMAIN)"
|
||
@echo " Documentation: https://docs.$(DOMAIN)"
|
||
@echo " Profiling: https://profiling.$(DOMAIN)"
|
||
@echo ""
|
||
@echo "📡 Direct URLs:"
|
||
@echo " Vue.js Dev: http://localhost:8081"
|
||
@echo " Hugo Docs: http://localhost:1313"
|
||
|
||
.PHONY: dev-all
|
||
dev-all: dev-start dev-tools ## Start all services including development tools
|
||
|
||
##@ Management Commands
|
||
|
||
.PHONY: status
|
||
status: ## Show service status
|
||
@echo "📊 Service Status:"
|
||
@$(DOCKER_COMPOSE) ps
|
||
|
||
.PHONY: logs
|
||
logs: ## Show recent logs from all services
|
||
@$(DOCKER_COMPOSE) logs --tail=50
|
||
|
||
.PHONY: logs-follow
|
||
logs-follow: ## Follow logs from all services
|
||
@$(DOCKER_COMPOSE) logs -f
|
||
|
||
.PHONY: logs-manager
|
||
logs-manager: ## Show Manager logs
|
||
@$(DOCKER_COMPOSE) logs -f flamenco-manager
|
||
|
||
.PHONY: logs-worker
|
||
logs-worker: ## Show Worker logs
|
||
@$(DOCKER_COMPOSE) logs -f flamenco-worker
|
||
|
||
##@ Utility Commands
|
||
|
||
.PHONY: shell-manager
|
||
shell-manager: ## Open shell in Manager container
|
||
@$(DOCKER_COMPOSE) exec flamenco-manager bash
|
||
|
||
.PHONY: shell-worker
|
||
shell-worker: ## Open shell in Worker container
|
||
@$(DOCKER_COMPOSE) exec flamenco-worker bash
|
||
|
||
.PHONY: shell-tools
|
||
shell-tools: ## Open shell in dev-tools container
|
||
@$(DOCKER_COMPOSE_TOOLS) run --rm dev-tools bash
|
||
|
||
.PHONY: generate
|
||
generate: ## Regenerate API code in Manager container
|
||
@echo "🔄 Regenerating API code..."
|
||
@$(DOCKER_COMPOSE) exec flamenco-manager ./mage generate
|
||
@echo "✅ Code generation complete"
|
||
|
||
.PHONY: test
|
||
test: ## Run tests in Manager container
|
||
@echo "🧪 Running tests..."
|
||
@$(DOCKER_COMPOSE) exec flamenco-manager ./mage check
|
||
|
||
.PHONY: webapp-build
|
||
webapp-build: ## Build webapp static files
|
||
@echo "🏗️ Building webapp..."
|
||
@$(DOCKER_COMPOSE) exec flamenco-manager ./mage webappStatic
|
||
@echo "✅ Webapp build complete"
|
||
|
||
##@ Optimized Build Commands
|
||
|
||
.PHONY: build-optimized
|
||
build-optimized: ## Use optimized build with caching and parallelization
|
||
@echo "⚡ Running optimized build with caching..."
|
||
@$(DOCKER_COMPOSE) exec flamenco-manager ./mage buildOptimized
|
||
@echo "✅ Optimized build complete"
|
||
|
||
.PHONY: build-incremental
|
||
build-incremental: ## Use incremental build with caching
|
||
@echo "📈 Running incremental build..."
|
||
@$(DOCKER_COMPOSE) exec flamenco-manager ./mage buildIncremental
|
||
@echo "✅ Incremental build complete"
|
||
|
||
.PHONY: cache-status
|
||
cache-status: ## Show build cache status
|
||
@echo "📊 Build cache status:"
|
||
@$(DOCKER_COMPOSE) exec flamenco-manager ./mage cacheStatus
|
||
|
||
.PHONY: cache-clean
|
||
cache-clean: ## Clean build cache
|
||
@echo "🧹 Cleaning build cache..."
|
||
@$(DOCKER_COMPOSE) exec flamenco-manager ./mage cleanCache
|
||
@echo "✅ Build cache cleaned"
|
||
|
||
.PHONY: build-stats
|
||
build-stats: cache-status ## Show build performance statistics
|
||
|
||
##@ Database Commands
|
||
|
||
.PHONY: db-status
|
||
db-status: ## Show database migration status
|
||
@echo "🗄️ Database migration status:"
|
||
@$(DOCKER_COMPOSE_TOOLS) run --rm dev-tools goose -dir ./internal/manager/persistence/migrations/ sqlite3 /data/flamenco-manager.sqlite status
|
||
|
||
.PHONY: db-up
|
||
db-up: ## Apply database migrations
|
||
@echo "⬆️ Applying database migrations..."
|
||
@$(DOCKER_COMPOSE_TOOLS) run --rm dev-tools goose -dir ./internal/manager/persistence/migrations/ sqlite3 /data/flamenco-manager.sqlite up
|
||
@echo "✅ Database migrations applied"
|
||
|
||
.PHONY: db-down
|
||
db-down: ## Rollback database migrations
|
||
@echo "⬇️ Rolling back database migrations..."
|
||
@$(DOCKER_COMPOSE_TOOLS) run --rm dev-tools goose -dir ./internal/manager/persistence/migrations/ sqlite3 /data/flamenco-manager.sqlite down
|
||
@echo "✅ Database migration rolled back"
|
||
|
||
##@ Control Commands
|
||
|
||
.PHONY: dev-stop
|
||
dev-stop: ## Stop all services
|
||
@echo "🛑 Stopping all services..."
|
||
@$(DOCKER_COMPOSE) down
|
||
@echo "✅ All services stopped"
|
||
|
||
.PHONY: dev-restart
|
||
dev-restart: ## Restart all services
|
||
@echo "🔄 Restarting services..."
|
||
@$(DOCKER_COMPOSE) restart
|
||
@$(MAKE) status
|
||
|
||
.PHONY: dev-clean
|
||
dev-clean: ## Stop services and remove volumes
|
||
@echo "🧹 Cleaning development environment..."
|
||
@$(DOCKER_COMPOSE) down -v
|
||
@docker system prune -f
|
||
@echo "✅ Development environment cleaned"
|
||
|
||
.PHONY: dev-rebuild
|
||
dev-rebuild: ## Rebuild images and restart services
|
||
@echo "🔨 Rebuilding development environment..."
|
||
@$(DOCKER_COMPOSE) down
|
||
@docker compose --progress plain -f $(COMPOSE_FILE) build --no-cache
|
||
@$(MAKE) dev-start
|
||
@echo "✅ Development environment rebuilt"
|
||
|
||
##@ Production Commands
|
||
|
||
.PHONY: prod-build
|
||
prod-build: ## Build production images
|
||
@echo "🏭 Building production images..."
|
||
@docker build -f Dockerfile.dev --target production -t flamenco:latest .
|
||
@echo "✅ Production images built"
|
||
|
||
.PHONY: prod-run
|
||
prod-run: ## Run production container
|
||
@echo "🚀 Starting production container..."
|
||
@docker run -d \
|
||
--name flamenco-manager-prod \
|
||
-p 8080:8080 \
|
||
-v flamenco-prod-data:/data \
|
||
-v flamenco-prod-storage:/shared-storage \
|
||
flamenco:latest
|
||
|
||
##@ Configuration Commands
|
||
|
||
.PHONY: config
|
||
config: ## Show resolved compose configuration
|
||
@$(DOCKER_COMPOSE) config
|
||
|
||
.PHONY: config-validate
|
||
config-validate: ## Validate compose file syntax
|
||
@echo "✅ Validating compose file..."
|
||
@$(DOCKER_COMPOSE) config --quiet
|
||
@echo "✅ Compose file is valid"
|
||
|
||
.PHONY: env-show
|
||
env-show: ## Show current environment variables
|
||
@echo "📋 Environment Variables:"
|
||
@echo " COMPOSE_PROJECT_NAME: $(COMPOSE_PROJECT_NAME)"
|
||
@echo " DOMAIN: $(DOMAIN)"
|
||
@grep -E "^[A-Z_]+" .env 2>/dev/null || echo " (no .env file found)"
|
||
|
||
##@ Cleanup Commands
|
||
|
||
.PHONY: clean-volumes
|
||
clean-volumes: ## Remove all project volumes (DESTRUCTIVE)
|
||
@echo "⚠️ This will remove all data volumes!"
|
||
@read -p "Are you sure? [y/N] " -n 1 -r; \
|
||
echo ""; \
|
||
if [[ $$REPLY =~ ^[Yy]$$ ]]; then \
|
||
docker volume rm $(COMPOSE_PROJECT_NAME)-data $(COMPOSE_PROJECT_NAME)-shared-storage $(COMPOSE_PROJECT_NAME)-worker-data $(COMPOSE_PROJECT_NAME)-go-mod-cache $(COMPOSE_PROJECT_NAME)-yarn-cache 2>/dev/null || true; \
|
||
echo "✅ Volumes removed"; \
|
||
else \
|
||
echo "❌ Cancelled"; \
|
||
fi
|
||
|
||
.PHONY: clean-images
|
||
clean-images: ## Remove project images
|
||
@echo "🗑️ Removing project images..."
|
||
@docker images --filter "reference=$(COMPOSE_PROJECT_NAME)*" -q | xargs -r docker rmi
|
||
@echo "✅ Project images removed"
|
||
|
||
.PHONY: clean-all
|
||
clean-all: dev-stop clean-volumes clean-images ## Complete cleanup (DESTRUCTIVE)
|
||
@echo "✅ Complete cleanup finished"
|
||
|
||
# =============================================================================
|
||
# Development Shortcuts
|
||
# =============================================================================
|
||
|
||
.PHONY: up
|
||
up: dev-start ## Alias for dev-start
|
||
|
||
.PHONY: down
|
||
down: dev-stop ## Alias for dev-stop
|
||
|
||
.PHONY: ps
|
||
ps: status ## Alias for status
|
||
|
||
.PHONY: build
|
||
build: ## Build development images
|
||
@docker compose --progress plain -f $(COMPOSE_FILE) build
|
||
|
||
.PHONY: pull
|
||
pull: ## Pull latest base images
|
||
@$(DOCKER_COMPOSE) pull |