Add host agent with VM lifecycle, TAP networking, and envd client

Implements Phase 1: boot a Firecracker microVM, execute a command inside
it via envd, and get the output back. Uses raw Firecracker HTTP API via
Unix socket (not the Go SDK) for full control over the VM lifecycle.

- internal/vm: VM manager with create/pause/resume/destroy, Firecracker
  HTTP client, process launcher with unshare + ip netns exec isolation
- internal/network: per-sandbox network namespace with veth pair, TAP
  device, NAT rules, and IP forwarding
- internal/envdclient: Connect RPC client for envd process/filesystem
  services with health check retry
- cmd/host-agent: demo binary that boots a VM, runs "echo hello", prints
  output, and cleans up
- proto/envd: canonical proto files with buf + protoc-gen-connect-go
  code generation
- images/wrenn-init.sh: minimal PID 1 init script for guest VMs
- CLAUDE.md: updated architecture to reflect TAP networking (not vsock)
  and Firecracker HTTP API (not Go SDK)
This commit is contained in:
2026-03-10 00:06:47 +06:00
parent a3898d68fb
commit 7753938044
26 changed files with 5773 additions and 1275 deletions

View File

@ -0,0 +1,122 @@
package vm
import "fmt"
// VMConfig holds the configuration for creating a Firecracker microVM.
type VMConfig struct {
// SandboxID is the unique identifier for this sandbox (e.g., "sb-a1b2c3d4").
SandboxID string
// KernelPath is the path to the uncompressed Linux kernel (vmlinux).
KernelPath string
// RootfsPath is the path to the ext4 rootfs image for this sandbox.
// This should be a per-sandbox copy (reflink clone of the base image).
RootfsPath string
// VCPUs is the number of virtual CPUs to allocate (default: 1).
VCPUs int
// MemoryMB is the amount of RAM in megabytes (default: 512).
MemoryMB int
// NetworkNamespace is the name of the network namespace to launch
// Firecracker inside (e.g., "ns-1"). The namespace must already exist
// with a TAP device configured.
NetworkNamespace string
// TapDevice is the name of the TAP device inside the network namespace
// that Firecracker will attach to (e.g., "tap0").
TapDevice string
// TapMAC is the MAC address for the TAP device.
TapMAC string
// GuestIP is the IP address assigned to the guest VM (e.g., "169.254.0.21").
GuestIP string
// GatewayIP is the gateway IP (the TAP device's IP, e.g., "169.254.0.22").
GatewayIP string
// NetMask is the subnet mask for the guest network (e.g., "255.255.255.252").
NetMask string
// FirecrackerBin is the path to the firecracker binary.
FirecrackerBin string
// SocketPath is the path for the Firecracker API Unix socket.
SocketPath string
// SandboxDir is the tmpfs mount point for per-sandbox files inside the
// mount namespace (e.g., "/fc-vm").
SandboxDir string
// InitPath is the path to the init process inside the guest.
// Defaults to "/sbin/init" if empty.
InitPath string
}
func (c *VMConfig) applyDefaults() {
if c.VCPUs == 0 {
c.VCPUs = 1
}
if c.MemoryMB == 0 {
c.MemoryMB = 512
}
if c.FirecrackerBin == "" {
c.FirecrackerBin = "/usr/local/bin/firecracker"
}
if c.SocketPath == "" {
c.SocketPath = fmt.Sprintf("/tmp/fc-%s.sock", c.SandboxID)
}
if c.SandboxDir == "" {
c.SandboxDir = fmt.Sprintf("/tmp/fc-sandbox-%s", c.SandboxID)
}
if c.TapDevice == "" {
c.TapDevice = "tap0"
}
if c.TapMAC == "" {
c.TapMAC = "02:FC:00:00:00:05"
}
if c.InitPath == "" {
c.InitPath = "/usr/local/bin/wrenn-init"
}
}
// kernelArgs builds the kernel command line for the VM.
func (c *VMConfig) kernelArgs() string {
// ip= format: <client-ip>::<gw-ip>:<netmask>:<hostname>:<iface>:<autoconf>
ipArg := fmt.Sprintf("ip=%s::%s:%s:sandbox:eth0:off",
c.GuestIP, c.GatewayIP, c.NetMask,
)
return fmt.Sprintf(
"console=ttyS0 reboot=k panic=1 pci=off quiet loglevel=1 init=%s %s",
c.InitPath, ipArg,
)
}
func (c *VMConfig) validate() error {
if c.SandboxID == "" {
return fmt.Errorf("SandboxID is required")
}
if c.KernelPath == "" {
return fmt.Errorf("KernelPath is required")
}
if c.RootfsPath == "" {
return fmt.Errorf("RootfsPath is required")
}
if c.NetworkNamespace == "" {
return fmt.Errorf("NetworkNamespace is required")
}
if c.GuestIP == "" {
return fmt.Errorf("GuestIP is required")
}
if c.GatewayIP == "" {
return fmt.Errorf("GatewayIP is required")
}
if c.NetMask == "" {
return fmt.Errorf("NetMask is required")
}
return nil
}

141
internal/vm/fc.go Normal file
View File

@ -0,0 +1,141 @@
package vm
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"time"
)
// fcClient talks to the Firecracker HTTP API over a Unix socket.
type fcClient struct {
http *http.Client
socketPath string
}
func newFCClient(socketPath string) *fcClient {
return &fcClient{
socketPath: socketPath,
http: &http.Client{
Transport: &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "unix", socketPath)
},
},
Timeout: 10 * time.Second,
},
}
}
func (c *fcClient) do(ctx context.Context, method, path string, body any) error {
var bodyReader io.Reader
if body != nil {
data, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("marshal request body: %w", err)
}
bodyReader = bytes.NewReader(data)
}
// The host in the URL is ignored for Unix sockets; we use "localhost" by convention.
req, err := http.NewRequestWithContext(ctx, method, "http://localhost"+path, bodyReader)
if err != nil {
return fmt.Errorf("create request: %w", err)
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := c.http.Do(req)
if err != nil {
return fmt.Errorf("%s %s: %w", method, path, err)
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
respBody, _ := io.ReadAll(resp.Body)
return fmt.Errorf("%s %s: status %d: %s", method, path, resp.StatusCode, string(respBody))
}
return nil
}
// setBootSource configures the kernel and boot args.
func (c *fcClient) setBootSource(ctx context.Context, kernelPath, bootArgs string) error {
return c.do(ctx, http.MethodPut, "/boot-source", map[string]string{
"kernel_image_path": kernelPath,
"boot_args": bootArgs,
})
}
// setRootfsDrive configures the root filesystem drive.
func (c *fcClient) setRootfsDrive(ctx context.Context, driveID, path string, readOnly bool) error {
return c.do(ctx, http.MethodPut, "/drives/"+driveID, map[string]any{
"drive_id": driveID,
"path_on_host": path,
"is_root_device": true,
"is_read_only": readOnly,
})
}
// setNetworkInterface configures a network interface attached to a TAP device.
func (c *fcClient) setNetworkInterface(ctx context.Context, ifaceID, tapName, macAddr string) error {
return c.do(ctx, http.MethodPut, "/network-interfaces/"+ifaceID, map[string]any{
"iface_id": ifaceID,
"host_dev_name": tapName,
"guest_mac": macAddr,
})
}
// setMachineConfig configures vCPUs, memory, and other machine settings.
func (c *fcClient) setMachineConfig(ctx context.Context, vcpus, memMB int) error {
return c.do(ctx, http.MethodPut, "/machine-config", map[string]any{
"vcpu_count": vcpus,
"mem_size_mib": memMB,
"smt": false,
})
}
// startVM issues the InstanceStart action.
func (c *fcClient) startVM(ctx context.Context) error {
return c.do(ctx, http.MethodPut, "/actions", map[string]string{
"action_type": "InstanceStart",
})
}
// pauseVM pauses the microVM.
func (c *fcClient) pauseVM(ctx context.Context) error {
return c.do(ctx, http.MethodPatch, "/vm", map[string]string{
"state": "Paused",
})
}
// resumeVM resumes a paused microVM.
func (c *fcClient) resumeVM(ctx context.Context) error {
return c.do(ctx, http.MethodPatch, "/vm", map[string]string{
"state": "Resumed",
})
}
// createSnapshot creates a full VM snapshot.
func (c *fcClient) createSnapshot(ctx context.Context, snapPath, memPath string) error {
return c.do(ctx, http.MethodPut, "/snapshot/create", map[string]any{
"snapshot_type": "Full",
"snapshot_path": snapPath,
"mem_file_path": memPath,
})
}
// loadSnapshot loads a VM snapshot.
func (c *fcClient) loadSnapshot(ctx context.Context, snapPath, memPath string) error {
return c.do(ctx, http.MethodPut, "/snapshot/load", map[string]any{
"snapshot_path": snapPath,
"mem_file_path": memPath,
"resume_vm": false,
})
}

View File

@ -0,0 +1,125 @@
package vm
import (
"context"
"fmt"
"log/slog"
"os"
"os/exec"
"syscall"
"time"
)
// process represents a running Firecracker process with mount and network
// namespace isolation.
type process struct {
cmd *exec.Cmd
cancel context.CancelFunc
exitCh chan struct{}
exitErr error
}
// startProcess launches the Firecracker binary inside an isolated mount namespace
// and the specified network namespace. The launch sequence:
//
// 1. unshare -m: creates a private mount namespace
// 2. mount --make-rprivate /: prevents mount propagation to host
// 3. mount tmpfs at SandboxDir: ephemeral workspace for this VM
// 4. symlink kernel and rootfs into SandboxDir
// 5. ip netns exec <ns>: enters the network namespace where TAP is configured
// 6. exec firecracker with the API socket path
func startProcess(ctx context.Context, cfg *VMConfig) (*process, error) {
execCtx, cancel := context.WithCancel(ctx)
script := buildStartScript(cfg)
cmd := exec.CommandContext(execCtx, "unshare", "-m", "--", "bash", "-c", script)
cmd.SysProcAttr = &syscall.SysProcAttr{
Setsid: true, // new session so signals don't propagate from parent
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
cancel()
return nil, fmt.Errorf("start firecracker process: %w", err)
}
p := &process{
cmd: cmd,
cancel: cancel,
exitCh: make(chan struct{}),
}
go func() {
p.exitErr = cmd.Wait()
close(p.exitCh)
}()
slog.Info("firecracker process started",
"pid", cmd.Process.Pid,
"sandbox", cfg.SandboxID,
)
return p, nil
}
// buildStartScript generates the bash script that sets up the mount namespace,
// symlinks kernel/rootfs, and execs Firecracker inside the network namespace.
func buildStartScript(cfg *VMConfig) string {
return fmt.Sprintf(`
set -euo pipefail
# Prevent mount propagation to the host
mount --make-rprivate /
# Create ephemeral tmpfs workspace
mkdir -p %[1]s
mount -t tmpfs tmpfs %[1]s
# Symlink kernel and rootfs into the workspace
ln -s %[2]s %[1]s/vmlinux
ln -s %[3]s %[1]s/rootfs.ext4
# Launch Firecracker inside the network namespace
exec ip netns exec %[4]s %[5]s --api-sock %[6]s
`,
cfg.SandboxDir, // 1
cfg.KernelPath, // 2
cfg.RootfsPath, // 3
cfg.NetworkNamespace, // 4
cfg.FirecrackerBin, // 5
cfg.SocketPath, // 6
)
}
// stop sends SIGTERM and waits for the process to exit. If it doesn't exit
// within 10 seconds, SIGKILL is sent.
func (p *process) stop() error {
if p.cmd.Process == nil {
return nil
}
// Send SIGTERM to the process group (negative PID).
if err := syscall.Kill(-p.cmd.Process.Pid, syscall.SIGTERM); err != nil {
slog.Debug("sigterm failed, process may have exited", "error", err)
}
select {
case <-p.exitCh:
return nil
case <-time.After(10 * time.Second):
slog.Warn("firecracker did not exit after SIGTERM, sending SIGKILL")
if err := syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL); err != nil {
slog.Debug("sigkill failed", "error", err)
}
<-p.exitCh
return nil
}
}
// exited returns a channel that is closed when the process exits.
func (p *process) exited() <-chan struct{} {
return p.exitCh
}

View File

@ -0,0 +1,192 @@
package vm
import (
"context"
"fmt"
"log/slog"
"os"
"time"
)
// VM represents a running Firecracker microVM.
type VM struct {
Config VMConfig
process *process
client *fcClient
}
// Manager handles the lifecycle of Firecracker microVMs.
type Manager struct {
// vms tracks running VMs by sandbox ID.
vms map[string]*VM
}
// NewManager creates a new VM manager.
func NewManager() *Manager {
return &Manager{
vms: make(map[string]*VM),
}
}
// Create boots a new Firecracker microVM with the given configuration.
// The network namespace and TAP device must already be set up.
func (m *Manager) Create(ctx context.Context, cfg VMConfig) (*VM, error) {
cfg.applyDefaults()
if err := cfg.validate(); err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
// Clean up any leftover socket from a previous run.
os.Remove(cfg.SocketPath)
slog.Info("creating VM",
"sandbox", cfg.SandboxID,
"vcpus", cfg.VCPUs,
"memory_mb", cfg.MemoryMB,
)
// Step 1: Launch the Firecracker process.
proc, err := startProcess(ctx, &cfg)
if err != nil {
return nil, fmt.Errorf("start process: %w", err)
}
// Step 2: Wait for the API socket to appear.
if err := waitForSocket(ctx, cfg.SocketPath, proc); err != nil {
proc.stop()
return nil, fmt.Errorf("wait for socket: %w", err)
}
// Step 3: Configure the VM via the Firecracker API.
client := newFCClient(cfg.SocketPath)
if err := configureVM(ctx, client, &cfg); err != nil {
proc.stop()
return nil, fmt.Errorf("configure VM: %w", err)
}
// Step 4: Start the VM.
if err := client.startVM(ctx); err != nil {
proc.stop()
return nil, fmt.Errorf("start VM: %w", err)
}
vm := &VM{
Config: cfg,
process: proc,
client: client,
}
m.vms[cfg.SandboxID] = vm
slog.Info("VM started successfully", "sandbox", cfg.SandboxID)
return vm, nil
}
// configureVM sends the configuration to Firecracker via its HTTP API.
func configureVM(ctx context.Context, client *fcClient, cfg *VMConfig) error {
// Boot source (kernel + args)
if err := client.setBootSource(ctx, cfg.KernelPath, cfg.kernelArgs()); err != nil {
return fmt.Errorf("set boot source: %w", err)
}
// Root drive
if err := client.setRootfsDrive(ctx, "rootfs", cfg.RootfsPath, false); err != nil {
return fmt.Errorf("set rootfs drive: %w", err)
}
// Network interface
if err := client.setNetworkInterface(ctx, "eth0", cfg.TapDevice, cfg.TapMAC); err != nil {
return fmt.Errorf("set network interface: %w", err)
}
// Machine config (vCPUs + memory)
if err := client.setMachineConfig(ctx, cfg.VCPUs, cfg.MemoryMB); err != nil {
return fmt.Errorf("set machine config: %w", err)
}
return nil
}
// Pause pauses a running VM.
func (m *Manager) Pause(ctx context.Context, sandboxID string) error {
vm, ok := m.vms[sandboxID]
if !ok {
return fmt.Errorf("VM not found: %s", sandboxID)
}
if err := vm.client.pauseVM(ctx); err != nil {
return fmt.Errorf("pause VM: %w", err)
}
slog.Info("VM paused", "sandbox", sandboxID)
return nil
}
// Resume resumes a paused VM.
func (m *Manager) Resume(ctx context.Context, sandboxID string) error {
vm, ok := m.vms[sandboxID]
if !ok {
return fmt.Errorf("VM not found: %s", sandboxID)
}
if err := vm.client.resumeVM(ctx); err != nil {
return fmt.Errorf("resume VM: %w", err)
}
slog.Info("VM resumed", "sandbox", sandboxID)
return nil
}
// Destroy stops and cleans up a VM.
func (m *Manager) Destroy(ctx context.Context, sandboxID string) error {
vm, ok := m.vms[sandboxID]
if !ok {
return fmt.Errorf("VM not found: %s", sandboxID)
}
slog.Info("destroying VM", "sandbox", sandboxID)
// Stop the Firecracker process.
if err := vm.process.stop(); err != nil {
slog.Warn("error stopping process", "sandbox", sandboxID, "error", err)
}
// Clean up the API socket.
os.Remove(vm.Config.SocketPath)
delete(m.vms, sandboxID)
slog.Info("VM destroyed", "sandbox", sandboxID)
return nil
}
// Get returns a running VM by sandbox ID.
func (m *Manager) Get(sandboxID string) (*VM, bool) {
vm, ok := m.vms[sandboxID]
return vm, ok
}
// waitForSocket polls for the Firecracker API socket to appear on disk.
func waitForSocket(ctx context.Context, socketPath string, proc *process) error {
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
timeout := time.After(5 * time.Second)
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-proc.exited():
return fmt.Errorf("firecracker process exited before socket was ready")
case <-timeout:
return fmt.Errorf("timed out waiting for API socket at %s", socketPath)
case <-ticker.C:
if _, err := os.Stat(socketPath); err == nil {
return nil
}
}
}
}