Prototype with single host server and no admin panel (#2)

Reviewed-on: wrenn/sandbox#2 Co-authored-by: pptx704 <rafeed@omukk.dev> Co-committed-by: pptx704 <rafeed@omukk.dev>
2026-03-22 21:01:23 +00:00
parent bd78cc068c
commit 32e5a5a715
293 changed files with 46885 additions and 1033 deletions
--- a/internal/vm/config.go
+++ b/internal/vm/config.go
@ -0,0 +1,122 @@
+package vm
+
+import "fmt"
+
+// VMConfig holds the configuration for creating a Firecracker microVM.
+type VMConfig struct {
+	// SandboxID is the unique identifier for this sandbox (e.g., "sb-a1b2c3d4").
+	SandboxID string
+
+	// KernelPath is the path to the uncompressed Linux kernel (vmlinux).
+	KernelPath string
+
+	// RootfsPath is the path to the rootfs block device for this sandbox.
+	// Typically a dm-snapshot device (e.g., /dev/mapper/wrenn-sb-a1b2c3d4).
+	RootfsPath string
+
+	// VCPUs is the number of virtual CPUs to allocate (default: 1).
+	VCPUs int
+
+	// MemoryMB is the amount of RAM in megabytes (default: 512).
+	MemoryMB int
+
+	// NetworkNamespace is the name of the network namespace to launch
+	// Firecracker inside (e.g., "ns-1"). The namespace must already exist
+	// with a TAP device configured.
+	NetworkNamespace string
+
+	// TapDevice is the name of the TAP device inside the network namespace
+	// that Firecracker will attach to (e.g., "tap0").
+	TapDevice string
+
+	// TapMAC is the MAC address for the TAP device.
+	TapMAC string
+
+	// GuestIP is the IP address assigned to the guest VM (e.g., "169.254.0.21").
+	GuestIP string
+
+	// GatewayIP is the gateway IP (the TAP device's IP, e.g., "169.254.0.22").
+	GatewayIP string
+
+	// NetMask is the subnet mask for the guest network (e.g., "255.255.255.252").
+	NetMask string
+
+	// FirecrackerBin is the path to the firecracker binary.
+	FirecrackerBin string
+
+	// SocketPath is the path for the Firecracker API Unix socket.
+	SocketPath string
+
+	// SandboxDir is the tmpfs mount point for per-sandbox files inside the
+	// mount namespace (e.g., "/fc-vm").
+	SandboxDir string
+
+	// InitPath is the path to the init process inside the guest.
+	// Defaults to "/sbin/init" if empty.
+	InitPath string
+}
+
+func (c *VMConfig) applyDefaults() {
+	if c.VCPUs == 0 {
+		c.VCPUs = 1
+	}
+	if c.MemoryMB == 0 {
+		c.MemoryMB = 512
+	}
+	if c.FirecrackerBin == "" {
+		c.FirecrackerBin = "/usr/local/bin/firecracker"
+	}
+	if c.SocketPath == "" {
+		c.SocketPath = fmt.Sprintf("/tmp/fc-%s.sock", c.SandboxID)
+	}
+	if c.SandboxDir == "" {
+		c.SandboxDir = "/tmp/fc-vm"
+	}
+	if c.TapDevice == "" {
+		c.TapDevice = "tap0"
+	}
+	if c.TapMAC == "" {
+		c.TapMAC = "02:FC:00:00:00:05"
+	}
+	if c.InitPath == "" {
+		c.InitPath = "/usr/local/bin/wrenn-init"
+	}
+}
+
+// kernelArgs builds the kernel command line for the VM.
+func (c *VMConfig) kernelArgs() string {
+	// ip= format: <client-ip>::<gw-ip>:<netmask>:<hostname>:<iface>:<autoconf>
+	ipArg := fmt.Sprintf("ip=%s::%s:%s:sandbox:eth0:off",
+		c.GuestIP, c.GatewayIP, c.NetMask,
+	)
+
+	return fmt.Sprintf(
+		"console=ttyS0 reboot=k panic=1 pci=off quiet loglevel=1 init=%s %s",
+		c.InitPath, ipArg,
+	)
+}
+
+func (c *VMConfig) validate() error {
+	if c.SandboxID == "" {
+		return fmt.Errorf("SandboxID is required")
+	}
+	if c.KernelPath == "" {
+		return fmt.Errorf("KernelPath is required")
+	}
+	if c.RootfsPath == "" {
+		return fmt.Errorf("RootfsPath is required")
+	}
+	if c.NetworkNamespace == "" {
+		return fmt.Errorf("NetworkNamespace is required")
+	}
+	if c.GuestIP == "" {
+		return fmt.Errorf("GuestIP is required")
+	}
+	if c.GatewayIP == "" {
+		return fmt.Errorf("GatewayIP is required")
+	}
+	if c.NetMask == "" {
+		return fmt.Errorf("NetMask is required")
+	}
+	return nil
+}
--- a/internal/vm/fc.go
+++ b/internal/vm/fc.go
@ -0,0 +1,147 @@
+package vm
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"time"
+)
+
+// fcClient talks to the Firecracker HTTP API over a Unix socket.
+type fcClient struct {
+	http       *http.Client
+	socketPath string
+}
+
+func newFCClient(socketPath string) *fcClient {
+	return &fcClient{
+		socketPath: socketPath,
+		http: &http.Client{
+			Transport: &http.Transport{
+				DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+					var d net.Dialer
+					return d.DialContext(ctx, "unix", socketPath)
+				},
+			},
+			Timeout: 10 * time.Second,
+		},
+	}
+}
+
+func (c *fcClient) do(ctx context.Context, method, path string, body any) error {
+	var bodyReader io.Reader
+	if body != nil {
+		data, err := json.Marshal(body)
+		if err != nil {
+			return fmt.Errorf("marshal request body: %w", err)
+		}
+		bodyReader = bytes.NewReader(data)
+	}
+
+	// The host in the URL is ignored for Unix sockets; we use "localhost" by convention.
+	req, err := http.NewRequestWithContext(ctx, method, "http://localhost"+path, bodyReader)
+	if err != nil {
+		return fmt.Errorf("create request: %w", err)
+	}
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return fmt.Errorf("%s %s: %w", method, path, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 300 {
+		respBody, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("%s %s: status %d: %s", method, path, resp.StatusCode, string(respBody))
+	}
+
+	return nil
+}
+
+// setBootSource configures the kernel and boot args.
+func (c *fcClient) setBootSource(ctx context.Context, kernelPath, bootArgs string) error {
+	return c.do(ctx, http.MethodPut, "/boot-source", map[string]string{
+		"kernel_image_path": kernelPath,
+		"boot_args":         bootArgs,
+	})
+}
+
+// setRootfsDrive configures the root filesystem drive.
+func (c *fcClient) setRootfsDrive(ctx context.Context, driveID, path string, readOnly bool) error {
+	return c.do(ctx, http.MethodPut, "/drives/"+driveID, map[string]any{
+		"drive_id":       driveID,
+		"path_on_host":   path,
+		"is_root_device": true,
+		"is_read_only":   readOnly,
+	})
+}
+
+// setNetworkInterface configures a network interface attached to a TAP device.
+func (c *fcClient) setNetworkInterface(ctx context.Context, ifaceID, tapName, macAddr string) error {
+	return c.do(ctx, http.MethodPut, "/network-interfaces/"+ifaceID, map[string]any{
+		"iface_id":      ifaceID,
+		"host_dev_name": tapName,
+		"guest_mac":     macAddr,
+	})
+}
+
+// setMachineConfig configures vCPUs, memory, and other machine settings.
+func (c *fcClient) setMachineConfig(ctx context.Context, vcpus, memMB int) error {
+	return c.do(ctx, http.MethodPut, "/machine-config", map[string]any{
+		"vcpu_count":   vcpus,
+		"mem_size_mib": memMB,
+		"smt":          false,
+	})
+}
+
+// startVM issues the InstanceStart action.
+func (c *fcClient) startVM(ctx context.Context) error {
+	return c.do(ctx, http.MethodPut, "/actions", map[string]string{
+		"action_type": "InstanceStart",
+	})
+}
+
+// pauseVM pauses the microVM.
+func (c *fcClient) pauseVM(ctx context.Context) error {
+	return c.do(ctx, http.MethodPatch, "/vm", map[string]string{
+		"state": "Paused",
+	})
+}
+
+// resumeVM resumes a paused microVM.
+func (c *fcClient) resumeVM(ctx context.Context) error {
+	return c.do(ctx, http.MethodPatch, "/vm", map[string]string{
+		"state": "Resumed",
+	})
+}
+
+// createSnapshot creates a VM snapshot.
+// snapshotType is "Full" (all memory) or "Diff" (only dirty pages since last resume).
+func (c *fcClient) createSnapshot(ctx context.Context, snapPath, memPath, snapshotType string) error {
+	return c.do(ctx, http.MethodPut, "/snapshot/create", map[string]any{
+		"snapshot_type": snapshotType,
+		"snapshot_path": snapPath,
+		"mem_file_path": memPath,
+	})
+}
+
+// loadSnapshotWithUffd loads a VM snapshot using a UFFD socket for
+// lazy memory loading. Firecracker will connect to the socket and
+// send the uffd fd + memory region mappings.
+func (c *fcClient) loadSnapshotWithUffd(ctx context.Context, snapPath, uffdSocketPath string) error {
+	return c.do(ctx, http.MethodPut, "/snapshot/load", map[string]any{
+		"snapshot_path": snapPath,
+		"resume_vm":     false,
+		"mem_backend": map[string]any{
+			"backend_type": "Uffd",
+			"backend_path": uffdSocketPath,
+		},
+	})
+}
--- a/internal/vm/jailer.go
+++ b/internal/vm/jailer.go
@ -0,0 +1,128 @@
+package vm
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"syscall"
+	"time"
+)
+
+// process represents a running Firecracker process with mount and network
+// namespace isolation.
+type process struct {
+	cmd    *exec.Cmd
+	cancel context.CancelFunc
+
+	exitCh  chan struct{}
+	exitErr error
+}
+
+// startProcess launches the Firecracker binary inside an isolated mount namespace
+// and the specified network namespace. The launch sequence:
+//
+//  1. unshare -m: creates a private mount namespace
+//  2. mount --make-rprivate /: prevents mount propagation to host
+//  3. mount tmpfs at SandboxDir: ephemeral workspace for this VM
+//  4. symlink kernel and rootfs into SandboxDir
+//  5. ip netns exec <ns>: enters the network namespace where TAP is configured
+//  6. exec firecracker with the API socket path
+func startProcess(ctx context.Context, cfg *VMConfig) (*process, error) {
+	// Use a background context for the long-lived Firecracker process.
+	// The request context (ctx) is only used for the startup phase — we must
+	// not tie the VM's lifetime to the HTTP request that created it.
+	execCtx, cancel := context.WithCancel(context.Background())
+
+	script := buildStartScript(cfg)
+
+	cmd := exec.CommandContext(execCtx, "unshare", "-m", "--", "bash", "-c", script)
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setsid: true, // new session so signals don't propagate from parent
+	}
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Start(); err != nil {
+		cancel()
+		return nil, fmt.Errorf("start firecracker process: %w", err)
+	}
+
+	p := &process{
+		cmd:    cmd,
+		cancel: cancel,
+		exitCh: make(chan struct{}),
+	}
+
+	go func() {
+		p.exitErr = cmd.Wait()
+		close(p.exitCh)
+	}()
+
+	slog.Info("firecracker process started",
+		"pid", cmd.Process.Pid,
+		"sandbox", cfg.SandboxID,
+	)
+
+	return p, nil
+}
+
+// buildStartScript generates the bash script that sets up the mount namespace,
+// symlinks kernel/rootfs, and execs Firecracker inside the network namespace.
+func buildStartScript(cfg *VMConfig) string {
+	return fmt.Sprintf(`
+set -euo pipefail
+
+# Prevent mount propagation to the host
+mount --make-rprivate /
+
+# Create ephemeral tmpfs workspace
+mkdir -p %[1]s
+mount -t tmpfs tmpfs %[1]s
+
+# Symlink kernel and rootfs into the workspace
+ln -s %[2]s %[1]s/vmlinux
+ln -s %[3]s %[1]s/rootfs.ext4
+
+# Launch Firecracker inside the network namespace
+exec ip netns exec %[4]s %[5]s --api-sock %[6]s
+`,
+		cfg.SandboxDir,       // 1
+		cfg.KernelPath,       // 2
+		cfg.RootfsPath,       // 3
+		cfg.NetworkNamespace, // 4
+		cfg.FirecrackerBin,   // 5
+		cfg.SocketPath,       // 6
+	)
+}
+
+// stop sends SIGTERM and waits for the process to exit. If it doesn't exit
+// within 10 seconds, SIGKILL is sent.
+func (p *process) stop() error {
+	if p.cmd.Process == nil {
+		return nil
+	}
+
+	// Send SIGTERM to the process group (negative PID).
+	if err := syscall.Kill(-p.cmd.Process.Pid, syscall.SIGTERM); err != nil {
+		slog.Debug("sigterm failed, process may have exited", "error", err)
+	}
+
+	select {
+	case <-p.exitCh:
+		return nil
+	case <-time.After(10 * time.Second):
+		slog.Warn("firecracker did not exit after SIGTERM, sending SIGKILL")
+		if err := syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL); err != nil {
+			slog.Debug("sigkill failed", "error", err)
+		}
+		<-p.exitCh
+		return nil
+	}
+}
+
+// exited returns a channel that is closed when the process exits.
+func (p *process) exited() <-chan struct{} {
+	return p.exitCh
+}
--- a/internal/vm/manager.go
+++ b/internal/vm/manager.go
@ -0,0 +1,280 @@
+package vm
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"time"
+)
+
+// VM represents a running Firecracker microVM.
+type VM struct {
+	Config  VMConfig
+	process *process
+	client  *fcClient
+}
+
+// Manager handles the lifecycle of Firecracker microVMs.
+type Manager struct {
+	// vms tracks running VMs by sandbox ID.
+	vms map[string]*VM
+}
+
+// NewManager creates a new VM manager.
+func NewManager() *Manager {
+	return &Manager{
+		vms: make(map[string]*VM),
+	}
+}
+
+// Create boots a new Firecracker microVM with the given configuration.
+// The network namespace and TAP device must already be set up.
+func (m *Manager) Create(ctx context.Context, cfg VMConfig) (*VM, error) {
+	cfg.applyDefaults()
+	if err := cfg.validate(); err != nil {
+		return nil, fmt.Errorf("invalid config: %w", err)
+	}
+
+	// Clean up any leftover socket from a previous run.
+	os.Remove(cfg.SocketPath)
+
+	slog.Info("creating VM",
+		"sandbox", cfg.SandboxID,
+		"vcpus", cfg.VCPUs,
+		"memory_mb", cfg.MemoryMB,
+	)
+
+	// Step 1: Launch the Firecracker process.
+	proc, err := startProcess(ctx, &cfg)
+	if err != nil {
+		return nil, fmt.Errorf("start process: %w", err)
+	}
+
+	// Step 2: Wait for the API socket to appear.
+	if err := waitForSocket(ctx, cfg.SocketPath, proc); err != nil {
+		_ = proc.stop()
+		return nil, fmt.Errorf("wait for socket: %w", err)
+	}
+
+	// Step 3: Configure the VM via the Firecracker API.
+	client := newFCClient(cfg.SocketPath)
+
+	if err := configureVM(ctx, client, &cfg); err != nil {
+		_ = proc.stop()
+		return nil, fmt.Errorf("configure VM: %w", err)
+	}
+
+	// Step 4: Start the VM.
+	if err := client.startVM(ctx); err != nil {
+		_ = proc.stop()
+		return nil, fmt.Errorf("start VM: %w", err)
+	}
+
+	vm := &VM{
+		Config:  cfg,
+		process: proc,
+		client:  client,
+	}
+
+	m.vms[cfg.SandboxID] = vm
+
+	slog.Info("VM started successfully", "sandbox", cfg.SandboxID)
+
+	return vm, nil
+}
+
+// configureVM sends the configuration to Firecracker via its HTTP API.
+func configureVM(ctx context.Context, client *fcClient, cfg *VMConfig) error {
+	// Boot source (kernel + args)
+	if err := client.setBootSource(ctx, cfg.KernelPath, cfg.kernelArgs()); err != nil {
+		return fmt.Errorf("set boot source: %w", err)
+	}
+
+	// Root drive — use the symlink path inside the mount namespace so that
+	// snapshots record a stable path that works on restore.
+	rootfsSymlink := cfg.SandboxDir + "/rootfs.ext4"
+	if err := client.setRootfsDrive(ctx, "rootfs", rootfsSymlink, false); err != nil {
+		return fmt.Errorf("set rootfs drive: %w", err)
+	}
+
+	// Network interface
+	if err := client.setNetworkInterface(ctx, "eth0", cfg.TapDevice, cfg.TapMAC); err != nil {
+		return fmt.Errorf("set network interface: %w", err)
+	}
+
+	// Machine config (vCPUs + memory)
+	if err := client.setMachineConfig(ctx, cfg.VCPUs, cfg.MemoryMB); err != nil {
+		return fmt.Errorf("set machine config: %w", err)
+	}
+
+	return nil
+}
+
+// Pause pauses a running VM.
+func (m *Manager) Pause(ctx context.Context, sandboxID string) error {
+	vm, ok := m.vms[sandboxID]
+	if !ok {
+		return fmt.Errorf("VM not found: %s", sandboxID)
+	}
+
+	if err := vm.client.pauseVM(ctx); err != nil {
+		return fmt.Errorf("pause VM: %w", err)
+	}
+
+	slog.Info("VM paused", "sandbox", sandboxID)
+	return nil
+}
+
+// Resume resumes a paused VM.
+func (m *Manager) Resume(ctx context.Context, sandboxID string) error {
+	vm, ok := m.vms[sandboxID]
+	if !ok {
+		return fmt.Errorf("VM not found: %s", sandboxID)
+	}
+
+	if err := vm.client.resumeVM(ctx); err != nil {
+		return fmt.Errorf("resume VM: %w", err)
+	}
+
+	slog.Info("VM resumed", "sandbox", sandboxID)
+	return nil
+}
+
+// Destroy stops and cleans up a VM.
+func (m *Manager) Destroy(ctx context.Context, sandboxID string) error {
+	vm, ok := m.vms[sandboxID]
+	if !ok {
+		return fmt.Errorf("VM not found: %s", sandboxID)
+	}
+
+	slog.Info("destroying VM", "sandbox", sandboxID)
+
+	// Stop the Firecracker process.
+	if err := vm.process.stop(); err != nil {
+		slog.Warn("error stopping process", "sandbox", sandboxID, "error", err)
+	}
+
+	// Clean up the API socket.
+	os.Remove(vm.Config.SocketPath)
+
+	delete(m.vms, sandboxID)
+
+	slog.Info("VM destroyed", "sandbox", sandboxID)
+	return nil
+}
+
+// Snapshot creates a VM snapshot. The VM must already be paused.
+// snapshotType is "Full" (all memory) or "Diff" (only dirty pages since last resume).
+func (m *Manager) Snapshot(ctx context.Context, sandboxID, snapPath, memPath, snapshotType string) error {
+	vm, ok := m.vms[sandboxID]
+	if !ok {
+		return fmt.Errorf("VM not found: %s", sandboxID)
+	}
+
+	if err := vm.client.createSnapshot(ctx, snapPath, memPath, snapshotType); err != nil {
+		return fmt.Errorf("create snapshot: %w", err)
+	}
+
+	slog.Info("VM snapshot created", "sandbox", sandboxID, "snap_path", snapPath, "type", snapshotType)
+	return nil
+}
+
+// CreateFromSnapshot boots a new Firecracker VM by loading a snapshot
+// using UFFD for lazy memory loading. The network namespace and TAP
+// device must already be set up.
+//
+// No boot resources (kernel, drives, machine config) are configured —
+// the snapshot carries all that state. The rootfs path recorded in the
+// snapshot is resolved via a stable symlink at SandboxDir/rootfs.ext4
+// inside the mount namespace (created by the start script in jailer.go).
+//
+// The sequence is:
+//  1. Start FC process in mount+network namespace (creates tmpfs + rootfs symlink)
+//  2. Wait for API socket
+//  3. Load snapshot with UFFD backend
+//  4. Resume VM execution
+func (m *Manager) CreateFromSnapshot(ctx context.Context, cfg VMConfig, snapPath, uffdSocketPath string) (*VM, error) {
+	cfg.applyDefaults()
+	if err := cfg.validate(); err != nil {
+		return nil, fmt.Errorf("invalid config: %w", err)
+	}
+
+	os.Remove(cfg.SocketPath)
+
+	slog.Info("restoring VM from snapshot",
+		"sandbox", cfg.SandboxID,
+		"snap_path", snapPath,
+	)
+
+	// Step 1: Launch the Firecracker process.
+	// The start script creates a tmpfs at SandboxDir and symlinks
+	// rootfs.ext4 → cfg.RootfsPath, so the snapshot's recorded rootfs
+	// path (/fc-vm/rootfs.ext4) resolves to the new clone.
+	proc, err := startProcess(ctx, &cfg)
+	if err != nil {
+		return nil, fmt.Errorf("start process: %w", err)
+	}
+
+	// Step 2: Wait for the API socket.
+	if err := waitForSocket(ctx, cfg.SocketPath, proc); err != nil {
+		_ = proc.stop()
+		return nil, fmt.Errorf("wait for socket: %w", err)
+	}
+
+	client := newFCClient(cfg.SocketPath)
+
+	// Step 3: Load the snapshot with UFFD backend.
+	// No boot resources are configured — the snapshot carries kernel,
+	// drive, network, and machine config state.
+	if err := client.loadSnapshotWithUffd(ctx, snapPath, uffdSocketPath); err != nil {
+		_ = proc.stop()
+		return nil, fmt.Errorf("load snapshot: %w", err)
+	}
+
+	// Step 4: Resume the VM.
+	if err := client.resumeVM(ctx); err != nil {
+		_ = proc.stop()
+		return nil, fmt.Errorf("resume VM: %w", err)
+	}
+
+	vm := &VM{
+		Config:  cfg,
+		process: proc,
+		client:  client,
+	}
+
+	m.vms[cfg.SandboxID] = vm
+
+	slog.Info("VM restored from snapshot", "sandbox", cfg.SandboxID)
+	return vm, nil
+}
+
+// Get returns a running VM by sandbox ID.
+func (m *Manager) Get(sandboxID string) (*VM, bool) {
+	vm, ok := m.vms[sandboxID]
+	return vm, ok
+}
+
+// waitForSocket polls for the Firecracker API socket to appear on disk.
+func waitForSocket(ctx context.Context, socketPath string, proc *process) error {
+	ticker := time.NewTicker(10 * time.Millisecond)
+	defer ticker.Stop()
+
+	timeout := time.After(5 * time.Second)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-proc.exited():
+			return fmt.Errorf("firecracker process exited before socket was ready")
+		case <-timeout:
+			return fmt.Errorf("timed out waiting for API socket at %s", socketPath)
+		case <-ticker.C:
+			if _, err := os.Stat(socketPath); err == nil {
+				return nil
+			}
+		}
+	}
+}