Port envd from e2b with internalized shared packages and Connect RPC
- Copy envd source from e2b-dev/infra, internalize shared dependencies
into envd/internal/shared/ (keys, filesystem, id, smap, utils)
- Switch from gRPC to Connect RPC for all envd services
- Update module paths to git.omukk.dev/wrenn/{sandbox,sandbox/envd}
- Add proto specs (process, filesystem) with buf-based code generation
- Implement full envd: process exec, filesystem ops, port forwarding,
cgroup management, MMDS integration, and HTTP API
- Update main module dependencies (firecracker SDK, pgx, goose, etc.)
- Remove placeholder .gitkeep files replaced by real implementations
This commit is contained in:
14
envd/spec/buf.gen.yaml
Normal file
14
envd/spec/buf.gen.yaml
Normal file
@ -0,0 +1,14 @@
|
||||
version: v1
|
||||
plugins:
|
||||
- plugin: go
|
||||
out: ../internal/services/spec
|
||||
opt: paths=source_relative
|
||||
- plugin: connect-go
|
||||
out: ../internal/services/spec
|
||||
opt: paths=source_relative
|
||||
|
||||
managed:
|
||||
enabled: true
|
||||
optimize_for: SPEED
|
||||
go_package_prefix:
|
||||
default: git.omukk.dev/wrenn/sandbox/envd/internal/services/spec
|
||||
303
envd/spec/envd.yaml
Normal file
303
envd/spec/envd.yaml
Normal file
@ -0,0 +1,303 @@
|
||||
openapi: 3.0.0
|
||||
info:
|
||||
title: envd
|
||||
version: 0.1.1
|
||||
description: API for managing files' content and controlling envd
|
||||
|
||||
tags:
|
||||
- name: files
|
||||
|
||||
paths:
|
||||
/health:
|
||||
get:
|
||||
summary: Check the health of the service
|
||||
responses:
|
||||
"204":
|
||||
description: The service is healthy
|
||||
|
||||
/metrics:
|
||||
get:
|
||||
summary: Get the stats of the service
|
||||
security:
|
||||
- AccessTokenAuth: []
|
||||
- {}
|
||||
responses:
|
||||
"200":
|
||||
description: The resource usage metrics of the service
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Metrics"
|
||||
|
||||
/init:
|
||||
post:
|
||||
summary: Set initial vars, ensure the time and metadata is synced with the host
|
||||
security:
|
||||
- AccessTokenAuth: []
|
||||
- {}
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
volumeMounts:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/VolumeMount"
|
||||
hyperloopIP:
|
||||
type: string
|
||||
description: IP address of the hyperloop server to connect to
|
||||
envVars:
|
||||
$ref: "#/components/schemas/EnvVars"
|
||||
accessToken:
|
||||
type: string
|
||||
description: Access token for secure access to envd service
|
||||
x-go-type: SecureToken
|
||||
timestamp:
|
||||
type: string
|
||||
format: date-time
|
||||
description: The current timestamp in RFC3339 format
|
||||
defaultUser:
|
||||
type: string
|
||||
description: The default user to use for operations
|
||||
defaultWorkdir:
|
||||
type: string
|
||||
description: The default working directory to use for operations
|
||||
responses:
|
||||
"204":
|
||||
description: Env vars set, the time and metadata is synced with the host
|
||||
|
||||
/envs:
|
||||
get:
|
||||
summary: Get the environment variables
|
||||
security:
|
||||
- AccessTokenAuth: []
|
||||
- {}
|
||||
responses:
|
||||
"200":
|
||||
description: Environment variables
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/EnvVars"
|
||||
|
||||
/files:
|
||||
get:
|
||||
summary: Download a file
|
||||
tags: [files]
|
||||
security:
|
||||
- AccessTokenAuth: []
|
||||
- {}
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/FilePath"
|
||||
- $ref: "#/components/parameters/User"
|
||||
- $ref: "#/components/parameters/Signature"
|
||||
- $ref: "#/components/parameters/SignatureExpiration"
|
||||
responses:
|
||||
"200":
|
||||
$ref: "#/components/responses/DownloadSuccess"
|
||||
"401":
|
||||
$ref: "#/components/responses/InvalidUser"
|
||||
"400":
|
||||
$ref: "#/components/responses/InvalidPath"
|
||||
"404":
|
||||
$ref: "#/components/responses/FileNotFound"
|
||||
"500":
|
||||
$ref: "#/components/responses/InternalServerError"
|
||||
post:
|
||||
summary: Upload a file and ensure the parent directories exist. If the file exists, it will be overwritten.
|
||||
tags: [files]
|
||||
security:
|
||||
- AccessTokenAuth: []
|
||||
- {}
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/FilePath"
|
||||
- $ref: "#/components/parameters/User"
|
||||
- $ref: "#/components/parameters/Signature"
|
||||
- $ref: "#/components/parameters/SignatureExpiration"
|
||||
requestBody:
|
||||
$ref: "#/components/requestBodies/File"
|
||||
responses:
|
||||
"200":
|
||||
$ref: "#/components/responses/UploadSuccess"
|
||||
"400":
|
||||
$ref: "#/components/responses/InvalidPath"
|
||||
"401":
|
||||
$ref: "#/components/responses/InvalidUser"
|
||||
"500":
|
||||
$ref: "#/components/responses/InternalServerError"
|
||||
"507":
|
||||
$ref: "#/components/responses/NotEnoughDiskSpace"
|
||||
|
||||
components:
|
||||
securitySchemes:
|
||||
AccessTokenAuth:
|
||||
type: apiKey
|
||||
in: header
|
||||
name: X-Access-Token
|
||||
|
||||
parameters:
|
||||
FilePath:
|
||||
name: path
|
||||
in: query
|
||||
required: false
|
||||
description: Path to the file, URL encoded. Can be relative to user's home directory.
|
||||
schema:
|
||||
type: string
|
||||
User:
|
||||
name: username
|
||||
in: query
|
||||
required: false
|
||||
description: User used for setting the owner, or resolving relative paths.
|
||||
schema:
|
||||
type: string
|
||||
Signature:
|
||||
name: signature
|
||||
in: query
|
||||
required: false
|
||||
description: Signature used for file access permission verification.
|
||||
schema:
|
||||
type: string
|
||||
SignatureExpiration:
|
||||
name: signature_expiration
|
||||
in: query
|
||||
required: false
|
||||
description: Signature expiration used for defining the expiration time of the signature.
|
||||
schema:
|
||||
type: integer
|
||||
|
||||
requestBodies:
|
||||
File:
|
||||
required: true
|
||||
content:
|
||||
multipart/form-data:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
file:
|
||||
type: string
|
||||
format: binary
|
||||
|
||||
responses:
|
||||
UploadSuccess:
|
||||
description: The file was uploaded successfully.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/EntryInfo"
|
||||
|
||||
DownloadSuccess:
|
||||
description: Entire file downloaded successfully.
|
||||
content:
|
||||
application/octet-stream:
|
||||
schema:
|
||||
type: string
|
||||
format: binary
|
||||
description: The file content
|
||||
InvalidPath:
|
||||
description: Invalid path
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Error"
|
||||
InternalServerError:
|
||||
description: Internal server error
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Error"
|
||||
FileNotFound:
|
||||
description: File not found
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Error"
|
||||
InvalidUser:
|
||||
description: Invalid user
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Error"
|
||||
NotEnoughDiskSpace:
|
||||
description: Not enough disk space
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Error"
|
||||
|
||||
schemas:
|
||||
Error:
|
||||
required:
|
||||
- message
|
||||
- code
|
||||
properties:
|
||||
message:
|
||||
type: string
|
||||
description: Error message
|
||||
code:
|
||||
type: integer
|
||||
description: Error code
|
||||
EntryInfo:
|
||||
required:
|
||||
- path
|
||||
- name
|
||||
- type
|
||||
properties:
|
||||
path:
|
||||
type: string
|
||||
description: Path to the file
|
||||
name:
|
||||
type: string
|
||||
description: Name of the file
|
||||
type:
|
||||
type: string
|
||||
description: Type of the file
|
||||
enum:
|
||||
- file
|
||||
EnvVars:
|
||||
type: object
|
||||
description: Environment variables to set
|
||||
additionalProperties:
|
||||
type: string
|
||||
Metrics:
|
||||
type: object
|
||||
description: Resource usage metrics
|
||||
properties:
|
||||
ts:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Unix timestamp in UTC for current sandbox time
|
||||
cpu_count:
|
||||
type: integer
|
||||
description: Number of CPU cores
|
||||
cpu_used_pct:
|
||||
type: number
|
||||
format: float
|
||||
description: CPU usage percentage
|
||||
mem_total:
|
||||
type: integer
|
||||
description: Total virtual memory in bytes
|
||||
mem_used:
|
||||
type: integer
|
||||
description: Used virtual memory in bytes
|
||||
disk_used:
|
||||
type: integer
|
||||
description: Used disk space in bytes
|
||||
disk_total:
|
||||
type: integer
|
||||
description: Total disk space in bytes
|
||||
VolumeMount:
|
||||
type: object
|
||||
description: Volume
|
||||
additionalProperties: false
|
||||
properties:
|
||||
nfs_target:
|
||||
type: string
|
||||
path:
|
||||
type: string
|
||||
required:
|
||||
- nfs_target
|
||||
- path
|
||||
135
envd/spec/filesystem/filesystem.proto
Normal file
135
envd/spec/filesystem/filesystem.proto
Normal file
@ -0,0 +1,135 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package filesystem;
|
||||
|
||||
import "google/protobuf/timestamp.proto";
|
||||
|
||||
service Filesystem {
|
||||
rpc Stat(StatRequest) returns (StatResponse);
|
||||
rpc MakeDir(MakeDirRequest) returns (MakeDirResponse);
|
||||
rpc Move(MoveRequest) returns (MoveResponse);
|
||||
rpc ListDir(ListDirRequest) returns (ListDirResponse);
|
||||
rpc Remove(RemoveRequest) returns (RemoveResponse);
|
||||
|
||||
rpc WatchDir(WatchDirRequest) returns (stream WatchDirResponse);
|
||||
|
||||
// Non-streaming versions of WatchDir
|
||||
rpc CreateWatcher(CreateWatcherRequest) returns (CreateWatcherResponse);
|
||||
rpc GetWatcherEvents(GetWatcherEventsRequest) returns (GetWatcherEventsResponse);
|
||||
rpc RemoveWatcher(RemoveWatcherRequest) returns (RemoveWatcherResponse);
|
||||
}
|
||||
|
||||
message MoveRequest {
|
||||
string source = 1;
|
||||
string destination = 2;
|
||||
}
|
||||
|
||||
message MoveResponse {
|
||||
EntryInfo entry = 1;
|
||||
}
|
||||
|
||||
message MakeDirRequest {
|
||||
string path = 1;
|
||||
}
|
||||
|
||||
message MakeDirResponse {
|
||||
EntryInfo entry = 1;
|
||||
}
|
||||
|
||||
message RemoveRequest {
|
||||
string path = 1;
|
||||
}
|
||||
|
||||
message RemoveResponse {}
|
||||
|
||||
message StatRequest {
|
||||
string path = 1;
|
||||
}
|
||||
|
||||
message StatResponse {
|
||||
EntryInfo entry = 1;
|
||||
}
|
||||
|
||||
message EntryInfo {
|
||||
string name = 1;
|
||||
FileType type = 2;
|
||||
string path = 3;
|
||||
int64 size = 4;
|
||||
uint32 mode = 5;
|
||||
string permissions = 6;
|
||||
string owner = 7;
|
||||
string group = 8;
|
||||
google.protobuf.Timestamp modified_time = 9;
|
||||
// If the entry is a symlink, this field contains the target of the symlink.
|
||||
optional string symlink_target = 10;
|
||||
}
|
||||
|
||||
enum FileType {
|
||||
FILE_TYPE_UNSPECIFIED = 0;
|
||||
FILE_TYPE_FILE = 1;
|
||||
FILE_TYPE_DIRECTORY = 2;
|
||||
FILE_TYPE_SYMLINK = 3;
|
||||
}
|
||||
|
||||
message ListDirRequest {
|
||||
string path = 1;
|
||||
uint32 depth = 2;
|
||||
}
|
||||
|
||||
message ListDirResponse {
|
||||
repeated EntryInfo entries = 1;
|
||||
}
|
||||
|
||||
message WatchDirRequest {
|
||||
string path = 1;
|
||||
bool recursive = 2;
|
||||
}
|
||||
|
||||
message FilesystemEvent {
|
||||
string name = 1;
|
||||
EventType type = 2;
|
||||
}
|
||||
|
||||
message WatchDirResponse {
|
||||
oneof event {
|
||||
StartEvent start = 1;
|
||||
FilesystemEvent filesystem = 2;
|
||||
KeepAlive keepalive = 3;
|
||||
}
|
||||
|
||||
message StartEvent {}
|
||||
|
||||
message KeepAlive {}
|
||||
}
|
||||
|
||||
message CreateWatcherRequest {
|
||||
string path = 1;
|
||||
bool recursive = 2;
|
||||
}
|
||||
|
||||
message CreateWatcherResponse {
|
||||
string watcher_id = 1;
|
||||
}
|
||||
|
||||
message GetWatcherEventsRequest {
|
||||
string watcher_id = 1;
|
||||
}
|
||||
|
||||
message GetWatcherEventsResponse {
|
||||
repeated FilesystemEvent events = 1;
|
||||
}
|
||||
|
||||
message RemoveWatcherRequest {
|
||||
string watcher_id = 1;
|
||||
}
|
||||
|
||||
message RemoveWatcherResponse {}
|
||||
|
||||
enum EventType {
|
||||
EVENT_TYPE_UNSPECIFIED = 0;
|
||||
EVENT_TYPE_CREATE = 1;
|
||||
EVENT_TYPE_WRITE = 2;
|
||||
EVENT_TYPE_REMOVE = 3;
|
||||
EVENT_TYPE_RENAME = 4;
|
||||
EVENT_TYPE_CHMOD = 5;
|
||||
}
|
||||
3
envd/spec/generate.go
Normal file
3
envd/spec/generate.go
Normal file
@ -0,0 +1,3 @@
|
||||
package spec
|
||||
|
||||
//go:generate buf generate --template buf.gen.yaml
|
||||
171
envd/spec/process/process.proto
Normal file
171
envd/spec/process/process.proto
Normal file
@ -0,0 +1,171 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package process;
|
||||
|
||||
service Process {
|
||||
rpc List(ListRequest) returns (ListResponse);
|
||||
|
||||
rpc Connect(ConnectRequest) returns (stream ConnectResponse);
|
||||
rpc Start(StartRequest) returns (stream StartResponse);
|
||||
|
||||
rpc Update(UpdateRequest) returns (UpdateResponse);
|
||||
|
||||
// Client input stream ensures ordering of messages
|
||||
rpc StreamInput(stream StreamInputRequest) returns (StreamInputResponse);
|
||||
rpc SendInput(SendInputRequest) returns (SendInputResponse);
|
||||
rpc SendSignal(SendSignalRequest) returns (SendSignalResponse);
|
||||
|
||||
// Close stdin to signal EOF to the process.
|
||||
// Only works for non-PTY processes. For PTY, send Ctrl+D (0x04) instead.
|
||||
rpc CloseStdin(CloseStdinRequest) returns (CloseStdinResponse);
|
||||
}
|
||||
|
||||
message PTY {
|
||||
Size size = 1;
|
||||
|
||||
message Size {
|
||||
uint32 cols = 1;
|
||||
uint32 rows = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message ProcessConfig {
|
||||
string cmd = 1;
|
||||
repeated string args = 2;
|
||||
|
||||
map<string, string> envs = 3;
|
||||
optional string cwd = 4;
|
||||
}
|
||||
|
||||
message ListRequest {}
|
||||
|
||||
message ProcessInfo {
|
||||
ProcessConfig config = 1;
|
||||
uint32 pid = 2;
|
||||
optional string tag = 3;
|
||||
}
|
||||
|
||||
message ListResponse {
|
||||
repeated ProcessInfo processes = 1;
|
||||
}
|
||||
|
||||
message StartRequest {
|
||||
ProcessConfig process = 1;
|
||||
optional PTY pty = 2;
|
||||
optional string tag = 3;
|
||||
// This is optional for backwards compatibility.
|
||||
// We default to true. New SDK versions will set this to false by default.
|
||||
optional bool stdin = 4;
|
||||
}
|
||||
|
||||
message UpdateRequest {
|
||||
ProcessSelector process = 1;
|
||||
|
||||
optional PTY pty = 2;
|
||||
}
|
||||
|
||||
message UpdateResponse {}
|
||||
|
||||
message ProcessEvent {
|
||||
oneof event {
|
||||
StartEvent start = 1;
|
||||
DataEvent data = 2;
|
||||
EndEvent end = 3;
|
||||
KeepAlive keepalive = 4;
|
||||
}
|
||||
|
||||
message StartEvent {
|
||||
uint32 pid = 1;
|
||||
}
|
||||
|
||||
message DataEvent {
|
||||
oneof output {
|
||||
bytes stdout = 1;
|
||||
bytes stderr = 2;
|
||||
bytes pty = 3;
|
||||
}
|
||||
}
|
||||
|
||||
message EndEvent {
|
||||
sint32 exit_code = 1;
|
||||
bool exited = 2;
|
||||
string status = 3;
|
||||
optional string error = 4;
|
||||
}
|
||||
|
||||
message KeepAlive {}
|
||||
}
|
||||
|
||||
message StartResponse {
|
||||
ProcessEvent event = 1;
|
||||
}
|
||||
|
||||
message ConnectResponse {
|
||||
ProcessEvent event = 1;
|
||||
}
|
||||
|
||||
message SendInputRequest {
|
||||
ProcessSelector process = 1;
|
||||
|
||||
ProcessInput input = 2;
|
||||
}
|
||||
|
||||
message SendInputResponse {}
|
||||
|
||||
message ProcessInput {
|
||||
oneof input {
|
||||
bytes stdin = 1;
|
||||
bytes pty = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message StreamInputRequest {
|
||||
oneof event {
|
||||
StartEvent start = 1;
|
||||
DataEvent data = 2;
|
||||
KeepAlive keepalive = 3;
|
||||
}
|
||||
|
||||
message StartEvent {
|
||||
ProcessSelector process = 1;
|
||||
}
|
||||
|
||||
message DataEvent {
|
||||
ProcessInput input = 2;
|
||||
}
|
||||
|
||||
message KeepAlive {}
|
||||
}
|
||||
|
||||
message StreamInputResponse {}
|
||||
|
||||
enum Signal {
|
||||
SIGNAL_UNSPECIFIED = 0;
|
||||
SIGNAL_SIGTERM = 15;
|
||||
SIGNAL_SIGKILL = 9;
|
||||
}
|
||||
|
||||
message SendSignalRequest {
|
||||
ProcessSelector process = 1;
|
||||
|
||||
Signal signal = 2;
|
||||
}
|
||||
|
||||
message SendSignalResponse {}
|
||||
|
||||
message CloseStdinRequest {
|
||||
ProcessSelector process = 1;
|
||||
}
|
||||
|
||||
message CloseStdinResponse {}
|
||||
|
||||
message ConnectRequest {
|
||||
ProcessSelector process = 1;
|
||||
}
|
||||
|
||||
message ProcessSelector {
|
||||
oneof selector {
|
||||
uint32 pid = 1;
|
||||
string tag = 2;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user