forked from wrenn/wrenn
Replace one-shot clock_settime with chrony for continuous guest time sync
Switch from the envd /init endpoint pushing host time via syscall to chronyd reading the KVM PTP hardware clock (/dev/ptp0) continuously. This fixes clock drift between init calls and handles snapshot resume gracefully. Changes: - Add clocksource=kvm-clock kernel boot arg - Start chronyd in wrenn-init.sh before tini (PHC /dev/ptp0, makestep 1.0 -1) - Remove clock_settime logic from envd SetData and shouldSetSystemTime - Remove client.Init() clock sync calls from sandbox manager (3 sites) - Remove Init() method from envdclient (no longer needed) - Simplify rootfs scripts: socat/chrony now come from apt in the container image, only envd/wrenn-init/tini are injected by build scripts
This commit is contained in:
@ -17,8 +17,6 @@ import (
|
||||
"github.com/awnumar/memguard"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/txn2/txeh"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"git.omukk.dev/wrenn/sandbox/envd/internal/host"
|
||||
"git.omukk.dev/wrenn/sandbox/envd/internal/logs"
|
||||
"git.omukk.dev/wrenn/sandbox/envd/internal/shared/keys"
|
||||
@ -29,11 +27,6 @@ var (
|
||||
ErrAccessTokenResetNotAuthorized = errors.New("access token reset not authorized")
|
||||
)
|
||||
|
||||
const (
|
||||
maxTimeInPast = 50 * time.Millisecond
|
||||
maxTimeInFuture = 5 * time.Second
|
||||
)
|
||||
|
||||
// validateInitAccessToken validates the access token for /init requests.
|
||||
// Token is valid if it matches the existing token OR the MMDS hash.
|
||||
// If neither exists, first-time setup is allowed.
|
||||
@ -172,20 +165,6 @@ func (a *API) SetData(ctx context.Context, logger zerolog.Logger, data PostInitJ
|
||||
return err
|
||||
}
|
||||
|
||||
if data.Timestamp != nil {
|
||||
// Check if current time differs significantly from the received timestamp
|
||||
if shouldSetSystemTime(time.Now(), *data.Timestamp) {
|
||||
logger.Debug().Msgf("Setting sandbox start time to: %v", *data.Timestamp)
|
||||
ts := unix.NsecToTimespec(data.Timestamp.UnixNano())
|
||||
err := unix.ClockSettime(unix.CLOCK_REALTIME, &ts)
|
||||
if err != nil {
|
||||
logger.Error().Msgf("Failed to set system time: %v", err)
|
||||
}
|
||||
} else {
|
||||
logger.Debug().Msgf("Current time is within acceptable range of timestamp %v, not setting system time", *data.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
if data.EnvVars != nil {
|
||||
logger.Debug().Msg(fmt.Sprintf("Setting %d env vars", len(*data.EnvVars)))
|
||||
|
||||
@ -309,9 +288,3 @@ func getIPFamily(address string) (txeh.IPFamily, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// shouldSetSystemTime returns true if the current time differs significantly from the received timestamp,
|
||||
// indicating the system clock should be adjusted. Returns true when the sandboxTime is more than
|
||||
// maxTimeInPast before the hostTime or more than maxTimeInFuture after the hostTime.
|
||||
func shouldSetSystemTime(sandboxTime, hostTime time.Time) bool {
|
||||
return sandboxTime.Before(hostTime.Add(-maxTimeInPast)) || sandboxTime.After(hostTime.Add(maxTimeInFuture))
|
||||
}
|
||||
|
||||
@ -9,7 +9,6 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@ -59,71 +58,6 @@ func TestSimpleCases(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldSetSystemTime(t *testing.T) {
|
||||
t.Parallel()
|
||||
sandboxTime := time.Now()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
hostTime time.Time
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "sandbox time far ahead of host time (should set)",
|
||||
hostTime: sandboxTime.Add(-10 * time.Second),
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "sandbox time at maxTimeInPast boundary ahead of host time (should not set)",
|
||||
hostTime: sandboxTime.Add(-50 * time.Millisecond),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "sandbox time just within maxTimeInPast ahead of host time (should not set)",
|
||||
hostTime: sandboxTime.Add(-40 * time.Millisecond),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "sandbox time slightly ahead of host time (should not set)",
|
||||
hostTime: sandboxTime.Add(-10 * time.Millisecond),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "sandbox time equals host time (should not set)",
|
||||
hostTime: sandboxTime,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "sandbox time slightly behind host time (should not set)",
|
||||
hostTime: sandboxTime.Add(1 * time.Second),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "sandbox time just within maxTimeInFuture behind host time (should not set)",
|
||||
hostTime: sandboxTime.Add(4 * time.Second),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "sandbox time at maxTimeInFuture boundary behind host time (should not set)",
|
||||
hostTime: sandboxTime.Add(5 * time.Second),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "sandbox time far behind host time (should set)",
|
||||
hostTime: sandboxTime.Add(1 * time.Minute),
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := shouldSetSystemTime(tt.hostTime, sandboxTime)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func secureTokenPtr(s string) *SecureToken {
|
||||
token := &SecureToken{}
|
||||
_ = token.Set([]byte(s))
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#!/bin/sh
|
||||
# wrenn-init: minimal PID 1 init for Firecracker microVMs.
|
||||
# Mounts virtual filesystems then execs envd.
|
||||
# Mounts virtual filesystems, starts chronyd for time sync, then execs tini + envd.
|
||||
|
||||
set -e
|
||||
|
||||
@ -27,5 +27,17 @@ echo "nameserver 8.8.4.4" >> /etc/resolv.conf
|
||||
# Set a standard PATH so envd and all child processes can find common binaries.
|
||||
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
# Write chrony config to sync time from the KVM PTP hardware clock.
|
||||
# /dev/ptp0 is a paravirtual clock exposed by KVM — no network required.
|
||||
mkdir -p /etc/chrony /run/chrony
|
||||
cat > /etc/chrony/chrony.conf <<EOF
|
||||
refclock PHC /dev/ptp0 poll 2 dpoll 2
|
||||
driftfile /run/chrony/chrony.drift
|
||||
makestep 1.0 -1
|
||||
EOF
|
||||
|
||||
# Start chronyd in the background before handing off to tini.
|
||||
chronyd -f /etc/chrony/chrony.conf 2>/dev/null || true
|
||||
|
||||
# Exec tini as PID 1 — it reaps zombie processes and forwards signals to envd.
|
||||
exec /sbin/tini -- /usr/local/bin/envd
|
||||
|
||||
@ -3,14 +3,12 @@ package envdclient
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"connectrpc.com/connect"
|
||||
|
||||
@ -49,35 +47,6 @@ func (c *Client) BaseURL() string {
|
||||
return c.base
|
||||
}
|
||||
|
||||
// Init calls POST /init on envd to sync the guest clock with the host.
|
||||
// This is important after snapshot resume where the guest clock is frozen.
|
||||
func (c *Client) Init(ctx context.Context) error {
|
||||
now := time.Now().UTC()
|
||||
body, err := json.Marshal(map[string]any{"timestamp": now})
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal init body: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/init", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create init request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("init: status %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExecResult holds the output of a command execution.
|
||||
type ExecResult struct {
|
||||
Stdout []byte
|
||||
|
||||
@ -203,16 +203,6 @@ func (m *Manager) Create(ctx context.Context, sandboxID, template string, vcpus,
|
||||
return nil, fmt.Errorf("wait for envd: %w", err)
|
||||
}
|
||||
|
||||
// Sync guest clock in background. Non-fatal — sandbox is usable before this completes.
|
||||
// Run in a goroutine so Init latency doesn't block the RPC response back to the control plane.
|
||||
go func() {
|
||||
initCtx, initCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer initCancel()
|
||||
if err := client.Init(initCtx); err != nil {
|
||||
slog.Warn("envd init (clock sync) failed", "sandbox", sandboxID, "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
now := time.Now()
|
||||
sb := &sandboxState{
|
||||
Sandbox: models.Sandbox{
|
||||
@ -636,16 +626,6 @@ func (m *Manager) Resume(ctx context.Context, sandboxID string, timeoutSec int)
|
||||
return nil, fmt.Errorf("wait for envd: %w", err)
|
||||
}
|
||||
|
||||
// Sync guest clock in background. Non-fatal — sandbox is usable before this completes.
|
||||
// Run in a goroutine so Init latency doesn't block the RPC response back to the control plane.
|
||||
go func() {
|
||||
initCtx, initCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer initCancel()
|
||||
if err := client.Init(initCtx); err != nil {
|
||||
slog.Warn("envd init (clock sync) failed", "sandbox", sandboxID, "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
now := time.Now()
|
||||
sb := &sandboxState{
|
||||
Sandbox: models.Sandbox{
|
||||
@ -957,16 +937,6 @@ func (m *Manager) createFromSnapshot(ctx context.Context, sandboxID, snapshotNam
|
||||
return nil, fmt.Errorf("wait for envd: %w", err)
|
||||
}
|
||||
|
||||
// Sync guest clock in background. Non-fatal — sandbox is usable before this completes.
|
||||
// Run in a goroutine so Init latency doesn't block the RPC response back to the control plane.
|
||||
go func() {
|
||||
initCtx, initCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer initCancel()
|
||||
if err := client.Init(initCtx); err != nil {
|
||||
slog.Warn("envd init (clock sync) failed", "sandbox", sandboxID, "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
now := time.Now()
|
||||
sb := &sandboxState{
|
||||
Sandbox: models.Sandbox{
|
||||
|
||||
@ -91,7 +91,7 @@ func (c *VMConfig) kernelArgs() string {
|
||||
)
|
||||
|
||||
return fmt.Sprintf(
|
||||
"console=ttyS0 reboot=k panic=1 pci=off quiet loglevel=1 init=%s %s",
|
||||
"console=ttyS0 reboot=k panic=1 pci=off quiet loglevel=1 clocksource=kvm-clock init=%s %s",
|
||||
c.InitPath, ipArg,
|
||||
)
|
||||
}
|
||||
|
||||
@ -3,7 +3,10 @@
|
||||
# rootfs-from-container.sh — Create a bootable Wrenn rootfs from a Docker container.
|
||||
#
|
||||
# Exports a container's filesystem, writes it into an ext4 image, injects
|
||||
# envd + wrenn-init, and shrinks the image to minimum size.
|
||||
# envd + wrenn-init + tini, and shrinks the image to minimum size.
|
||||
#
|
||||
# The container image must already include: socat, chrony, curl, ca-certificates, git.
|
||||
# These are installed via apt in the container before export, not injected here.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/rootfs-from-container.sh <container> <image_name>
|
||||
@ -15,8 +18,7 @@
|
||||
# Output:
|
||||
# ${AGENT_FILES_ROOTDIR}/images/<image_name>/rootfs.ext4
|
||||
#
|
||||
# Requires: docker, mkfs.ext4, resize2fs, e2fsck, make (for building envd), curl (for tini/socat download),
|
||||
# gcc, make (for building socat from source)
|
||||
# Requires: docker, mkfs.ext4, resize2fs, e2fsck, make (for building envd), curl (for tini download)
|
||||
# Sudo is used only for mount/umount/copy-into-image operations.
|
||||
|
||||
set -euo pipefail
|
||||
@ -131,45 +133,23 @@ sudo mkdir -p "${MOUNT_DIR}/sbin"
|
||||
sudo cp "${TINI_BIN}" "${MOUNT_DIR}/sbin/tini"
|
||||
sudo chmod 755 "${MOUNT_DIR}/sbin/tini"
|
||||
|
||||
echo "==> Installing socat..."
|
||||
SOCAT_BIN=""
|
||||
# 1. Already in the exported container image?
|
||||
for p in "${MOUNT_DIR}/usr/bin/socat" "${MOUNT_DIR}/usr/local/bin/socat"; do
|
||||
if [ -f "$p" ]; then SOCAT_BIN="$p"; break; fi
|
||||
done
|
||||
# 2. Available on the host?
|
||||
if [ -z "${SOCAT_BIN}" ]; then
|
||||
for p in /usr/bin/socat /usr/local/bin/socat; do
|
||||
if [ -f "$p" ]; then SOCAT_BIN="$p"; break; fi
|
||||
done
|
||||
fi
|
||||
# 3. Build from source.
|
||||
if [ -z "${SOCAT_BIN}" ]; then
|
||||
SOCAT_VERSION="1.8.1.1"
|
||||
SOCAT_URL="http://www.dest-unreach.org/socat/download/socat-${SOCAT_VERSION}.tar.gz"
|
||||
SOCAT_BUILD_DIR="/tmp/socat-build"
|
||||
echo " Building socat ${SOCAT_VERSION} from source..."
|
||||
rm -rf "${SOCAT_BUILD_DIR}"
|
||||
mkdir -p "${SOCAT_BUILD_DIR}"
|
||||
curl -fsSL "${SOCAT_URL}" | tar xz -C "${SOCAT_BUILD_DIR}" --strip-components=1
|
||||
(cd "${SOCAT_BUILD_DIR}" && LDFLAGS="-static" ./configure --quiet && make -j"$(nproc)" -s)
|
||||
SOCAT_BIN="${SOCAT_BUILD_DIR}/socat"
|
||||
if [ ! -f "${SOCAT_BIN}" ]; then
|
||||
echo "ERROR: socat build failed"
|
||||
exit 1
|
||||
fi
|
||||
if ! file "${SOCAT_BIN}" | grep -q "statically linked"; then
|
||||
echo "ERROR: socat is not statically linked!"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
sudo cp "${SOCAT_BIN}" "${MOUNT_DIR}/usr/local/bin/socat"
|
||||
sudo chmod 755 "${MOUNT_DIR}/usr/local/bin/socat"
|
||||
|
||||
# Step 7: Verify.
|
||||
# Step 6: Verify injected binaries and required container packages.
|
||||
echo ""
|
||||
echo "==> Installed guest binaries:"
|
||||
ls -la "${MOUNT_DIR}/usr/local/bin/envd" "${MOUNT_DIR}/usr/local/bin/wrenn-init" "${MOUNT_DIR}/sbin/tini" "${MOUNT_DIR}/usr/local/bin/socat"
|
||||
ls -la "${MOUNT_DIR}/usr/local/bin/envd" "${MOUNT_DIR}/usr/local/bin/wrenn-init" "${MOUNT_DIR}/sbin/tini"
|
||||
|
||||
echo ""
|
||||
echo "==> Checking required container packages..."
|
||||
MISSING_PKGS=""
|
||||
for bin in socat chronyd curl git; do
|
||||
if ! find "${MOUNT_DIR}" -name "${bin}" -type f 2>/dev/null | head -1 | grep -q .; then
|
||||
MISSING_PKGS="${MISSING_PKGS} ${bin}"
|
||||
fi
|
||||
done
|
||||
if [ -n "${MISSING_PKGS}" ]; then
|
||||
echo "WARNING: The following binaries were not found in the container image:${MISSING_PKGS}"
|
||||
echo " Install them in the container (via apt) before exporting."
|
||||
fi
|
||||
|
||||
# Unmount before shrinking.
|
||||
sudo umount "${MOUNT_DIR}"
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# update-debug-rootfs.sh — Build envd and inject it (plus wrenn-init) into the debug rootfs.
|
||||
# update-debug-rootfs.sh — Build envd and inject it (plus wrenn-init + tini) into the debug rootfs.
|
||||
#
|
||||
# This script:
|
||||
# 1. Builds a fresh envd static binary via make
|
||||
# 2. Mounts the rootfs image
|
||||
# 3. Copies envd and wrenn-init into the image
|
||||
# 3. Copies envd, wrenn-init, and tini into the image
|
||||
# 4. Unmounts cleanly
|
||||
#
|
||||
# Usage:
|
||||
@ -96,45 +96,10 @@ sudo mkdir -p "${MOUNT_DIR}/sbin"
|
||||
sudo cp "${TINI_BIN}" "${MOUNT_DIR}/sbin/tini"
|
||||
sudo chmod 755 "${MOUNT_DIR}/sbin/tini"
|
||||
|
||||
echo "==> Installing socat..."
|
||||
SOCAT_BIN=""
|
||||
# 1. Already in the rootfs?
|
||||
for p in "${MOUNT_DIR}/usr/bin/socat" "${MOUNT_DIR}/usr/local/bin/socat"; do
|
||||
if [ -f "$p" ]; then SOCAT_BIN="$p"; break; fi
|
||||
done
|
||||
# 2. Available on the host?
|
||||
if [ -z "${SOCAT_BIN}" ]; then
|
||||
for p in /usr/bin/socat /usr/local/bin/socat; do
|
||||
if [ -f "$p" ]; then SOCAT_BIN="$p"; break; fi
|
||||
done
|
||||
fi
|
||||
# 3. Build from source.
|
||||
if [ -z "${SOCAT_BIN}" ]; then
|
||||
SOCAT_VERSION="1.8.1.1"
|
||||
SOCAT_URL="http://www.dest-unreach.org/socat/download/socat-${SOCAT_VERSION}.tar.gz"
|
||||
SOCAT_BUILD_DIR="/tmp/socat-build"
|
||||
echo " Building socat ${SOCAT_VERSION} from source..."
|
||||
rm -rf "${SOCAT_BUILD_DIR}"
|
||||
mkdir -p "${SOCAT_BUILD_DIR}"
|
||||
curl -fsSL "${SOCAT_URL}" | tar xz -C "${SOCAT_BUILD_DIR}" --strip-components=1
|
||||
(cd "${SOCAT_BUILD_DIR}" && LDFLAGS="-static" ./configure --quiet && make -j"$(nproc)" -s)
|
||||
SOCAT_BIN="${SOCAT_BUILD_DIR}/socat"
|
||||
if [ ! -f "${SOCAT_BIN}" ]; then
|
||||
echo "ERROR: socat build failed"
|
||||
exit 1
|
||||
fi
|
||||
if ! file "${SOCAT_BIN}" | grep -q "statically linked"; then
|
||||
echo "ERROR: socat is not statically linked!"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
sudo cp "${SOCAT_BIN}" "${MOUNT_DIR}/usr/local/bin/socat"
|
||||
sudo chmod 755 "${MOUNT_DIR}/usr/local/bin/socat"
|
||||
|
||||
# Step 4: Verify.
|
||||
echo ""
|
||||
echo "==> Installed files:"
|
||||
ls -la "${MOUNT_DIR}/usr/local/bin/envd" "${MOUNT_DIR}/usr/local/bin/wrenn-init" "${MOUNT_DIR}/sbin/tini" "${MOUNT_DIR}/usr/local/bin/socat"
|
||||
ls -la "${MOUNT_DIR}/usr/local/bin/envd" "${MOUNT_DIR}/usr/local/bin/wrenn-init" "${MOUNT_DIR}/sbin/tini"
|
||||
|
||||
echo ""
|
||||
echo "==> Done. Rootfs updated: ${ROOTFS}"
|
||||
|
||||
Reference in New Issue
Block a user