Port envd from e2b with internalized shared packages and Connect RPC
- Copy envd source from e2b-dev/infra, internalize shared dependencies
into envd/internal/shared/ (keys, filesystem, id, smap, utils)
- Switch from gRPC to Connect RPC for all envd services
- Update module paths to git.omukk.dev/wrenn/{sandbox,sandbox/envd}
- Add proto specs (process, filesystem) with buf-based code generation
- Implement full envd: process exec, filesystem ops, port forwarding,
cgroup management, MMDS integration, and HTTP API
- Update main module dependencies (firecracker SDK, pgx, goose, etc.)
- Remove placeholder .gitkeep files replaced by real implementations
This commit is contained in:
93
envd/internal/host/metrics.go
Normal file
93
envd/internal/host/metrics.go
Normal file
@ -0,0 +1,93 @@
|
||||
package host
|
||||
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/shirou/gopsutil/v4/cpu"
|
||||
"github.com/shirou/gopsutil/v4/mem"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
type Metrics struct {
|
||||
Timestamp int64 `json:"ts"` // Unix Timestamp in UTC
|
||||
|
||||
CPUCount uint32 `json:"cpu_count"` // Total CPU cores
|
||||
CPUUsedPercent float32 `json:"cpu_used_pct"` // Percent rounded to 2 decimal places
|
||||
|
||||
// Deprecated: kept for backwards compatibility with older orchestrators.
|
||||
MemTotalMiB uint64 `json:"mem_total_mib"` // Total virtual memory in MiB
|
||||
|
||||
// Deprecated: kept for backwards compatibility with older orchestrators.
|
||||
MemUsedMiB uint64 `json:"mem_used_mib"` // Used virtual memory in MiB
|
||||
|
||||
MemTotal uint64 `json:"mem_total"` // Total virtual memory in bytes
|
||||
MemUsed uint64 `json:"mem_used"` // Used virtual memory in bytes
|
||||
|
||||
DiskUsed uint64 `json:"disk_used"` // Used disk space in bytes
|
||||
DiskTotal uint64 `json:"disk_total"` // Total disk space in bytes
|
||||
}
|
||||
|
||||
func GetMetrics() (*Metrics, error) {
|
||||
v, err := mem.VirtualMemory()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
memUsedMiB := v.Used / 1024 / 1024
|
||||
memTotalMiB := v.Total / 1024 / 1024
|
||||
|
||||
cpuTotal, err := cpu.Counts(true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cpuUsedPcts, err := cpu.Percent(0, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cpuUsedPct := cpuUsedPcts[0]
|
||||
cpuUsedPctRounded := float32(cpuUsedPct)
|
||||
if cpuUsedPct > 0 {
|
||||
cpuUsedPctRounded = float32(math.Round(cpuUsedPct*100) / 100)
|
||||
}
|
||||
|
||||
diskMetrics, err := diskStats("/")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Metrics{
|
||||
Timestamp: time.Now().UTC().Unix(),
|
||||
CPUCount: uint32(cpuTotal),
|
||||
CPUUsedPercent: cpuUsedPctRounded,
|
||||
MemUsedMiB: memUsedMiB,
|
||||
MemTotalMiB: memTotalMiB,
|
||||
MemTotal: v.Total,
|
||||
MemUsed: v.Used,
|
||||
DiskUsed: diskMetrics.Total - diskMetrics.Available,
|
||||
DiskTotal: diskMetrics.Total,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type diskSpace struct {
|
||||
Total uint64
|
||||
Available uint64
|
||||
}
|
||||
|
||||
func diskStats(path string) (diskSpace, error) {
|
||||
var st unix.Statfs_t
|
||||
if err := unix.Statfs(path, &st); err != nil {
|
||||
return diskSpace{}, err
|
||||
}
|
||||
|
||||
block := uint64(st.Bsize)
|
||||
|
||||
// all data blocks
|
||||
total := st.Blocks * block
|
||||
// blocks available
|
||||
available := st.Bavail * block
|
||||
|
||||
return diskSpace{Total: total, Available: available}, nil
|
||||
}
|
||||
182
envd/internal/host/mmds.go
Normal file
182
envd/internal/host/mmds.go
Normal file
@ -0,0 +1,182 @@
|
||||
package host
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"git.omukk.dev/wrenn/sandbox/envd/internal/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
WrennRunDir = "/run/wrenn" // store sandbox metadata files here
|
||||
|
||||
mmdsDefaultAddress = "169.254.169.254"
|
||||
mmdsTokenExpiration = 60 * time.Second
|
||||
|
||||
mmdsAccessTokenRequestClientTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
var mmdsAccessTokenClient = &http.Client{
|
||||
Timeout: mmdsAccessTokenRequestClientTimeout,
|
||||
Transport: &http.Transport{
|
||||
DisableKeepAlives: true,
|
||||
},
|
||||
}
|
||||
|
||||
type MMDSOpts struct {
|
||||
SandboxID string `json:"instanceID"`
|
||||
TemplateID string `json:"envID"`
|
||||
LogsCollectorAddress string `json:"address"`
|
||||
AccessTokenHash string `json:"accessTokenHash"`
|
||||
}
|
||||
|
||||
func (opts *MMDSOpts) Update(sandboxID, templateID, collectorAddress string) {
|
||||
opts.SandboxID = sandboxID
|
||||
opts.TemplateID = templateID
|
||||
opts.LogsCollectorAddress = collectorAddress
|
||||
}
|
||||
|
||||
func (opts *MMDSOpts) AddOptsToJSON(jsonLogs []byte) ([]byte, error) {
|
||||
parsed := make(map[string]any)
|
||||
|
||||
err := json.Unmarshal(jsonLogs, &parsed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parsed["instanceID"] = opts.SandboxID
|
||||
parsed["envID"] = opts.TemplateID
|
||||
|
||||
data, err := json.Marshal(parsed)
|
||||
|
||||
return data, err
|
||||
}
|
||||
|
||||
func getMMDSToken(ctx context.Context, client *http.Client) (string, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://"+mmdsDefaultAddress+"/latest/api/token", &bytes.Buffer{})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
request.Header["X-metadata-token-ttl-seconds"] = []string{fmt.Sprint(mmdsTokenExpiration.Seconds())}
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
token := string(body)
|
||||
|
||||
if len(token) == 0 {
|
||||
return "", fmt.Errorf("mmds token is an empty string")
|
||||
}
|
||||
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func getMMDSOpts(ctx context.Context, client *http.Client, token string) (*MMDSOpts, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+mmdsDefaultAddress, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
request.Header["X-metadata-token"] = []string{token}
|
||||
request.Header["Accept"] = []string{"application/json"}
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer response.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var opts MMDSOpts
|
||||
|
||||
err = json.Unmarshal(body, &opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &opts, nil
|
||||
}
|
||||
|
||||
// GetAccessTokenHashFromMMDS reads the access token hash from MMDS.
|
||||
// This is used to validate that /init requests come from the orchestrator.
|
||||
func GetAccessTokenHashFromMMDS(ctx context.Context) (string, error) {
|
||||
token, err := getMMDSToken(ctx, mmdsAccessTokenClient)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get MMDS token: %w", err)
|
||||
}
|
||||
|
||||
opts, err := getMMDSOpts(ctx, mmdsAccessTokenClient, token)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get MMDS opts: %w", err)
|
||||
}
|
||||
|
||||
return opts.AccessTokenHash, nil
|
||||
}
|
||||
|
||||
func PollForMMDSOpts(ctx context.Context, mmdsChan chan<- *MMDSOpts, envVars *utils.Map[string, string]) {
|
||||
httpClient := &http.Client{}
|
||||
defer httpClient.CloseIdleConnections()
|
||||
|
||||
ticker := time.NewTicker(50 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
fmt.Fprintf(os.Stderr, "context cancelled while waiting for mmds opts")
|
||||
|
||||
return
|
||||
case <-ticker.C:
|
||||
token, err := getMMDSToken(ctx, httpClient)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error getting mmds token: %v\n", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
mmdsOpts, err := getMMDSOpts(ctx, httpClient, token)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error getting mmds opts: %v\n", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
envVars.Store("WRENN_SANDBOX_ID", mmdsOpts.SandboxID)
|
||||
envVars.Store("WRENN_TEMPLATE_ID", mmdsOpts.TemplateID)
|
||||
|
||||
if err := os.WriteFile(filepath.Join(WrennRunDir, ".WRENN_SANDBOX_ID"), []byte(mmdsOpts.SandboxID), 0o666); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error writing sandbox ID file: %v\n", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(WrennRunDir, ".WRENN_TEMPLATE_ID"), []byte(mmdsOpts.TemplateID), 0o666); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error writing template ID file: %v\n", err)
|
||||
}
|
||||
|
||||
if mmdsOpts.LogsCollectorAddress != "" {
|
||||
mmdsChan <- mmdsOpts
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user