forked from wrenn/wrenn
v0.1.0 (#17)
This commit is contained in:
@ -1,191 +1,15 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
import "git.omukk.dev/wrenn/wrenn/pkg/cpserver"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"git.omukk.dev/wrenn/wrenn/internal/api"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/audit"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/auth"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/auth/oauth"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/channels"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/config"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/db"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/lifecycle"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/scheduler"
|
||||
// Set via -ldflags at build time.
|
||||
var (
|
||||
version = "dev"
|
||||
commit = "unknown"
|
||||
)
|
||||
|
||||
func main() {
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
||||
Level: slog.LevelDebug,
|
||||
})))
|
||||
|
||||
cfg := config.Load()
|
||||
|
||||
if len(cfg.JWTSecret) < 32 {
|
||||
slog.Error("JWT_SECRET must be at least 32 characters")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Database connection pool.
|
||||
pool, err := pgxpool.New(ctx, cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
slog.Error("failed to connect to database", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
slog.Error("failed to ping database", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
slog.Info("connected to database")
|
||||
|
||||
queries := db.New(pool)
|
||||
|
||||
// Redis client.
|
||||
redisOpts, err := redis.ParseURL(cfg.RedisURL)
|
||||
if err != nil {
|
||||
slog.Error("failed to parse REDIS_URL", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
rdb := redis.NewClient(redisOpts)
|
||||
defer rdb.Close()
|
||||
|
||||
if err := rdb.Ping(ctx).Err(); err != nil {
|
||||
slog.Error("failed to ping redis", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
slog.Info("connected to redis")
|
||||
|
||||
// mTLS is mandatory — parse internal CA for CP↔agent communication.
|
||||
if cfg.CACert == "" || cfg.CAKey == "" {
|
||||
slog.Error("WRENN_CA_CERT and WRENN_CA_KEY are required — mTLS is mandatory for CP↔agent communication")
|
||||
os.Exit(1)
|
||||
}
|
||||
ca, err := auth.ParseCA(cfg.CACert, cfg.CAKey)
|
||||
if err != nil {
|
||||
slog.Error("failed to parse mTLS CA from environment", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
slog.Info("mTLS enabled: CA loaded")
|
||||
|
||||
// Host client pool — manages Connect RPC clients to host agents.
|
||||
cpCertStore, err := auth.NewCPCertStore(ca)
|
||||
if err != nil {
|
||||
slog.Error("failed to issue CP client certificate", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
// Renew the CP client certificate periodically so it never expires
|
||||
// while the control plane is running (TTL = 24h, renewal = every 12h).
|
||||
go func() {
|
||||
ticker := time.NewTicker(auth.CPCertRenewInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := cpCertStore.Refresh(); err != nil {
|
||||
slog.Error("failed to renew CP client certificate", "error", err)
|
||||
} else {
|
||||
slog.Info("CP client certificate renewed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
hostPool := lifecycle.NewHostClientPoolTLS(auth.CPClientTLSConfig(ca, cpCertStore))
|
||||
slog.Info("host client pool: mTLS enabled")
|
||||
|
||||
// Scheduler — picks a host for each new sandbox (round-robin for now).
|
||||
hostScheduler := scheduler.NewRoundRobinScheduler(queries)
|
||||
|
||||
// OAuth provider registry.
|
||||
oauthRegistry := oauth.NewRegistry()
|
||||
if cfg.OAuthGitHubClientID != "" && cfg.OAuthGitHubClientSecret != "" {
|
||||
if cfg.CPPublicURL == "" {
|
||||
slog.Error("CP_PUBLIC_URL must be set when OAuth providers are configured")
|
||||
os.Exit(1)
|
||||
}
|
||||
callbackURL := strings.TrimRight(cfg.CPPublicURL, "/") + "/auth/oauth/github/callback"
|
||||
ghProvider := oauth.NewGitHubProvider(cfg.OAuthGitHubClientID, cfg.OAuthGitHubClientSecret, callbackURL)
|
||||
oauthRegistry.Register(ghProvider)
|
||||
slog.Info("registered OAuth provider", "provider", "github")
|
||||
}
|
||||
|
||||
// Channels: publisher, service, dispatcher.
|
||||
if len(cfg.EncryptionKeyHex) != 64 {
|
||||
slog.Error("WRENN_ENCRYPTION_KEY must be a hex-encoded 32-byte key (64 hex chars)")
|
||||
os.Exit(1)
|
||||
}
|
||||
channelPub := channels.NewPublisher(rdb)
|
||||
channelSvc := &channels.Service{DB: queries, EncKey: cfg.EncryptionKey}
|
||||
channelDispatcher := channels.NewDispatcher(rdb, queries, cfg.EncryptionKey)
|
||||
|
||||
// Shared audit logger with event publishing.
|
||||
al := audit.NewWithPublisher(queries, channelPub)
|
||||
|
||||
// API server.
|
||||
srv := api.New(queries, hostPool, hostScheduler, pool, rdb, []byte(cfg.JWTSecret), oauthRegistry, cfg.OAuthRedirectURL, ca, al, channelSvc)
|
||||
|
||||
// Start template build workers (2 concurrent).
|
||||
stopBuildWorkers := srv.BuildSvc.StartWorkers(ctx, 2)
|
||||
defer stopBuildWorkers()
|
||||
|
||||
// Start channel event dispatcher.
|
||||
channelDispatcher.Start(ctx)
|
||||
|
||||
// Start host monitor (passive + active reconciliation every 30s).
|
||||
monitor := api.NewHostMonitor(queries, hostPool, al, 30*time.Second)
|
||||
monitor.Start(ctx)
|
||||
|
||||
// Start metrics sampler (records per-team sandbox stats every 10s).
|
||||
sampler := api.NewMetricsSampler(queries, 10*time.Second)
|
||||
sampler.Start(ctx)
|
||||
|
||||
// Wrap the API handler with the sandbox proxy so that requests with
|
||||
// {port}-{sandbox_id}.{domain} Host headers are routed to the sandbox's
|
||||
// host agent. All other requests pass through to the normal API router.
|
||||
proxyWrapper := api.NewSandboxProxyWrapper(srv.Handler(), queries, hostPool)
|
||||
|
||||
httpServer := &http.Server{
|
||||
Addr: cfg.ListenAddr,
|
||||
Handler: proxyWrapper,
|
||||
}
|
||||
|
||||
// Graceful shutdown on signal.
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||
go func() {
|
||||
sig := <-sigCh
|
||||
slog.Info("received signal, shutting down", "signal", sig)
|
||||
cancel()
|
||||
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer shutdownCancel()
|
||||
|
||||
if err := httpServer.Shutdown(shutdownCtx); err != nil {
|
||||
slog.Error("http server shutdown error", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
slog.Info("control plane starting", "addr", cfg.ListenAddr)
|
||||
if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
slog.Error("http server error", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
slog.Info("control plane stopped")
|
||||
cpserver.Run(
|
||||
cpserver.WithVersion(version, commit),
|
||||
)
|
||||
}
|
||||
|
||||
@ -1,28 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
|
||||
"git.omukk.dev/wrenn/wrenn/internal/auth"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/devicemapper"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/hostagent"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/layout"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/network"
|
||||
"git.omukk.dev/wrenn/wrenn/internal/sandbox"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/auth"
|
||||
"git.omukk.dev/wrenn/wrenn/pkg/logging"
|
||||
"git.omukk.dev/wrenn/wrenn/proto/hostagent/gen/hostagentv1connect"
|
||||
)
|
||||
|
||||
// Set via -ldflags at build time.
|
||||
var (
|
||||
version = "dev"
|
||||
commit = "unknown"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Best-effort load — missing .env file is fine.
|
||||
_ = godotenv.Load()
|
||||
@ -31,18 +43,24 @@ func main() {
|
||||
advertiseAddr := flag.String("address", "", "Externally-reachable address (ip:port) for this host agent")
|
||||
flag.Parse()
|
||||
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
||||
Level: slog.LevelDebug,
|
||||
})))
|
||||
rootDir := envOrDefault("WRENN_DIR", "/var/lib/wrenn")
|
||||
cleanupLog := logging.Setup(filepath.Join(rootDir, "logs"), "host-agent")
|
||||
defer cleanupLog()
|
||||
|
||||
if os.Geteuid() != 0 {
|
||||
slog.Error("host agent must run as root")
|
||||
if err := checkPrivileges(); err != nil {
|
||||
slog.Error("insufficient privileges", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Enable IP forwarding (required for NAT).
|
||||
// Enable IP forwarding (required for NAT). The write may fail if running
|
||||
// as non-root without DAC_OVERRIDE on this path — that's OK if the systemd
|
||||
// unit's ExecStartPre already set it. We verify the value regardless.
|
||||
if err := os.WriteFile("/proc/sys/net/ipv4/ip_forward", []byte("1"), 0644); err != nil {
|
||||
slog.Warn("failed to enable ip_forward", "error", err)
|
||||
slog.Warn("failed to enable ip_forward (may have been set by systemd unit)", "error", err)
|
||||
}
|
||||
if b, err := os.ReadFile("/proc/sys/net/ipv4/ip_forward"); err != nil || strings.TrimSpace(string(b)) != "1" {
|
||||
slog.Error("ip_forward is not enabled — sandbox networking will be broken", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Clean up stale resources from a previous crash.
|
||||
@ -50,7 +68,6 @@ func main() {
|
||||
network.CleanupStaleNamespaces()
|
||||
|
||||
listenAddr := envOrDefault("WRENN_HOST_LISTEN_ADDR", ":50051")
|
||||
rootDir := envOrDefault("WRENN_DIR", "/var/lib/wrenn")
|
||||
cpURL := os.Getenv("WRENN_CP_URL")
|
||||
credsFile := filepath.Join(rootDir, "host-credentials.json")
|
||||
|
||||
@ -63,15 +80,50 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Expand base images to the standard disk size (sparse, no extra physical
|
||||
// Parse default rootfs size from env (e.g. "5G", "2Gi", "1000M").
|
||||
defaultRootfsSizeMB := sandbox.DefaultDiskSizeMB
|
||||
if sizeStr := os.Getenv("WRENN_DEFAULT_ROOTFS_SIZE"); sizeStr != "" {
|
||||
parsed, err := sandbox.ParseSizeToMB(sizeStr)
|
||||
if err != nil {
|
||||
slog.Error("invalid WRENN_DEFAULT_ROOTFS_SIZE", "value", sizeStr, "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defaultRootfsSizeMB = parsed
|
||||
slog.Info("using custom rootfs size", "size_mb", defaultRootfsSizeMB)
|
||||
}
|
||||
|
||||
// Expand base images to the configured disk size (sparse, no extra physical
|
||||
// disk). This ensures dm-snapshot sandboxes see the full size from boot.
|
||||
if err := sandbox.EnsureImageSizes(rootDir, sandbox.DefaultDiskSizeMB); err != nil {
|
||||
if err := sandbox.EnsureImageSizes(rootDir, defaultRootfsSizeMB); err != nil {
|
||||
slog.Error("failed to expand base images", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Resolve latest kernel version.
|
||||
kernelPath, kernelVersion, err := layout.LatestKernel(rootDir)
|
||||
if err != nil {
|
||||
slog.Error("failed to find kernel", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
slog.Info("resolved kernel", "version", kernelVersion, "path", kernelPath)
|
||||
|
||||
// Detect firecracker version.
|
||||
fcBin := envOrDefault("WRENN_FIRECRACKER_BIN", "/usr/local/bin/firecracker")
|
||||
fcVersion, err := sandbox.DetectFirecrackerVersion(fcBin)
|
||||
if err != nil {
|
||||
slog.Error("failed to detect firecracker version", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
slog.Info("resolved firecracker", "version", fcVersion, "path", fcBin)
|
||||
|
||||
cfg := sandbox.Config{
|
||||
WrennDir: rootDir,
|
||||
WrennDir: rootDir,
|
||||
DefaultRootfsSizeMB: defaultRootfsSizeMB,
|
||||
KernelPath: kernelPath,
|
||||
KernelVersion: kernelVersion,
|
||||
FirecrackerBin: fcBin,
|
||||
FirecrackerVersion: fcVersion,
|
||||
AgentVersion: version,
|
||||
}
|
||||
|
||||
mgr := sandbox.New(cfg)
|
||||
@ -128,6 +180,7 @@ func main() {
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer shutdownCancel()
|
||||
mgr.Shutdown(shutdownCtx)
|
||||
sandbox.ShrinkMinimalImage(rootDir)
|
||||
if err := httpServer.Shutdown(shutdownCtx); err != nil {
|
||||
slog.Error("http server shutdown error", "error", err)
|
||||
}
|
||||
@ -180,7 +233,7 @@ func main() {
|
||||
doShutdown("signal: " + sig.String())
|
||||
}()
|
||||
|
||||
slog.Info("host agent starting", "addr", listenAddr, "host_id", creds.HostID)
|
||||
slog.Info("host agent starting", "addr", listenAddr, "host_id", creds.HostID, "version", version, "commit", commit)
|
||||
// TLSConfig is always set (mTLS is mandatory). Create the TLS listener
|
||||
// manually because ListenAndServeTLS requires on-disk cert/key paths
|
||||
// but we use GetCertificate callback for hot-swap support.
|
||||
@ -203,3 +256,63 @@ func envOrDefault(key, def string) string {
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// checkPrivileges verifies the process has the required Linux capabilities.
|
||||
// Always reads CapEff — even for root — because a root process inside a
|
||||
// restricted container (e.g. docker --cap-drop=all) may not have all caps.
|
||||
func checkPrivileges() error {
|
||||
capEff, err := readEffectiveCaps()
|
||||
if err != nil {
|
||||
return fmt.Errorf("read capabilities: %w", err)
|
||||
}
|
||||
|
||||
// All capabilities required by the host agent at runtime.
|
||||
required := []struct {
|
||||
bit uint
|
||||
name string
|
||||
}{
|
||||
{1, "CAP_DAC_OVERRIDE"}, // /dev/loop*, /dev/mapper/*, /dev/net/tun
|
||||
{5, "CAP_KILL"}, // SIGTERM/SIGKILL to Firecracker processes
|
||||
{12, "CAP_NET_ADMIN"}, // netlink, iptables, routing, TAP/veth
|
||||
{13, "CAP_NET_RAW"}, // raw sockets (iptables)
|
||||
{19, "CAP_SYS_PTRACE"}, // reading /proc/self/ns/net (netns.Get)
|
||||
{21, "CAP_SYS_ADMIN"}, // netns, mount ns, losetup, dmsetup
|
||||
{27, "CAP_MKNOD"}, // device-mapper node creation
|
||||
}
|
||||
|
||||
var missing []string
|
||||
for _, cap := range required {
|
||||
if capEff&(1<<cap.bit) == 0 {
|
||||
missing = append(missing, cap.name)
|
||||
}
|
||||
}
|
||||
|
||||
if len(missing) > 0 {
|
||||
return fmt.Errorf("missing capabilities: %s — run as root or apply setcap to the binary",
|
||||
strings.Join(missing, ", "))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readEffectiveCaps parses the CapEff bitmask from /proc/self/status.
|
||||
func readEffectiveCaps() (uint64, error) {
|
||||
f, err := os.Open("/proc/self/status")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if hexStr, ok := strings.CutPrefix(line, "CapEff:"); ok {
|
||||
return strconv.ParseUint(strings.TrimSpace(hexStr), 16, 64)
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return 0, fmt.Errorf("read /proc/self/status: %w", err)
|
||||
}
|
||||
return 0, fmt.Errorf("CapEff not found in /proc/self/status")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user