diff --git a/.gitignore b/.gitignore index ab3ce3a..d7a88a0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,7 @@ harness # linux image -images/fio-urunc/bzImage +images/*-urunc/bzImage # python cache diff --git a/experiment.yml b/experiment.yml index 90bbe45..5bd8e20 100644 --- a/experiment.yml +++ b/experiment.yml @@ -53,3 +53,10 @@ experiments: runtime: urunc snapshotter: overlayfs + network: + workloads: + default: + image: networkstatic/iperf3:latest + other: + - image: docker.io/jimjuniorb/iperf3-urunc:0.3 + runtime: urunc diff --git a/images/iperf3-urunc/bunnyfile b/images/iperf3-urunc/bunnyfile new file mode 100644 index 0000000..27cede6 --- /dev/null +++ b/images/iperf3-urunc/bunnyfile @@ -0,0 +1,22 @@ +#syntax=harbor.nbfc.io/nubificus/bunny:latest +version: v0.1 + +platforms: + framework: linux + monitor: qemu + architecture: x86 + +rootfs: + from: networkstatic/iperf3:latest + type: raw + include: + - from: harbor.nbfc.io/nubificus/urunit:latest + source: /urunit + destination: /urunit + +kernel: + from: local + path: bzImage + + +entrypoint: ["/urunit", "/usr/bin/iperf3"] \ No newline at end of file diff --git a/internal/cli/run.go b/internal/cli/run.go index 2ec9174..e794286 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -18,6 +18,7 @@ import ( harnessruntime "github.com/urunc-dev/evaluation_suite/internal/runtime" runtimeHTTPReadiness "github.com/urunc-dev/evaluation_suite/internal/runtime/httpreadiness" runtimeLifecycle "github.com/urunc-dev/evaluation_suite/internal/runtime/lifecycle" + runtimeNetwork "github.com/urunc-dev/evaluation_suite/internal/runtime/network" runtimeStorage "github.com/urunc-dev/evaluation_suite/internal/runtime/storage" ) @@ -62,6 +63,8 @@ func NewRunCommand() *cobra.Command { return err } + var adapterFactories []orchestrator.AdapterFactory + containerdClient, err := containerd.New("/run/containerd/containerd.sock") if err != nil { return err @@ -70,27 +73,28 @@ func NewRunCommand() *cobra.Command { containerdNamespace := namespaces.WithNamespace(cmd.Context(), "default") - lifecycleAdapterFactory := func(trial plan.Trial) (harnessruntime.Adapter, error) { - return runtimeLifecycle.NewAdapter(containerdClient, &containerdNamespace), nil - } - - storageAdapterFactory := func(trial plan.Trial) (harnessruntime.Adapter, error) { - return runtimeStorage.NewAdapter(containerdClient, &containerdNamespace), nil - } - - httpReadinessAdapterFactory := func(trial plan.Trial) (harnessruntime.Adapter, error) { - return runtimeHTTPReadiness.NewAdapter(), nil - } - - orch := orchestrator.New( - lifecycleAdapterFactory, - storageAdapterFactory, - httpReadinessAdapterFactory, + // append the adapters for different experiments + adapterFactories = append(adapterFactories, + func(trial plan.Trial) (harnessruntime.Adapter, error) { + return runtimeLifecycle.NewAdapter(containerdClient, &containerdNamespace), nil + }, + func(trial plan.Trial) (harnessruntime.Adapter, error) { + return runtimeStorage.NewAdapter(containerdClient, &containerdNamespace), nil + }, + func(trial plan.Trial) (harnessruntime.Adapter, error) { + return runtimeNetwork.NewAdapter(), nil + }, + func(trial plan.Trial) (harnessruntime.Adapter, error) { + return runtimeHTTPReadiness.NewAdapter(), nil + }, ) + orch := orchestrator.New(adapterFactories...) + result, err := orch.Run(cmd.Context(), p, orchestrator.Options{ RunID: runID, }) + if err != nil { return err } @@ -157,3 +161,12 @@ func NewRunCommand() *cobra.Command { return cmd } + +func planHasExperiment(p *plan.Plan, name string) bool { + for _, trial := range p.Trials { + if trial.ExperimentName == name { + return true + } + } + return false +} diff --git a/internal/doctor/checks.go b/internal/doctor/checks.go index 91a3fc6..18a7ac8 100644 --- a/internal/doctor/checks.go +++ b/internal/doctor/checks.go @@ -38,6 +38,10 @@ func Run( ) } + if hasNetworkExperiment(m) { + checkTool(report, "nerdctl", true) + } + checkHostPorts(report, m) checkHostPathVolumes(report, m) checkEnvironmentVisibility(ctx, report) @@ -45,6 +49,11 @@ func Run( return report, nil } +func hasNetworkExperiment(m *manifest.Manifest) bool { + _, ok := m.Experiments["network"] + return ok +} + func checkManifest(report *Report, m *manifest.Manifest) { if err := manifest.Validate(m); err != nil { report.Add(StatusFail, "manifest", err.Error()) diff --git a/internal/runtime/network/adapter.go b/internal/runtime/network/adapter.go new file mode 100644 index 0000000..e5640fa --- /dev/null +++ b/internal/runtime/network/adapter.go @@ -0,0 +1,395 @@ +package network + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log" + "os" + "os/exec" + "strings" + "syscall" + "time" + + harnessruntime "github.com/urunc-dev/evaluation_suite/internal/runtime" +) + +const defaultImage = "networkstatic/iperf3:latest" + +type commandRunner func(context.Context, ...string) ([]byte, []byte, error) + +// Adapter runs both ends of an iperf3 benchmark through the nerdctl CLI. +// It deliberately has no containerd client dependency. +type Adapter struct { + run commandRunner + serverName string + serverRunning bool + networkName string +} + +func NewAdapter() *Adapter { return &Adapter{run: runNerdctl} } + +func (a *Adapter) ExperimentName() string { return "network" } + +func (a *Adapter) Prepare(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + log.Printf( + "Preparing network benchmark trial=%s runtime=%s handler=%s image=%s\n", + tc.Trial.ID, + tc.Trial.RuntimeName, + tc.Trial.RuntimeHandler, + image(tc), + ) + startedAt := time.Now() + stdout, stderr, err := a.run(ctx, "pull", image(tc)) + result := stageResult( + harnessruntime.StagePrepare, + startedAt, + tc, + "Pull iperf3 image", + map[string]any{ + "stdout": strings.TrimSpace(string(stdout)), + "stderr": strings.TrimSpace(string(stderr)), + }) + if err != nil { + return result, commandError("pull iperf3 image", err, stdout, stderr) + } + return result, nil +} + +func (a *Adapter) CreateTask(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + log.Printf( + "Creating network benchmark trial=%s runtime=%s handler=%s image=%s\n", + tc.Trial.ID, + tc.Trial.RuntimeName, + tc.Trial.RuntimeHandler, + image(tc), + ) + + startedAt := time.Now() + + a.networkName = tc.Trial.ID + "-iperf3-network" + // 1. Create a dedicated network for the iperf3 server and client to communicate + stdout, stderr, err := a.run(ctx, "network", "create", a.networkName) + if err != nil { + result := stageResult( + harnessruntime.StageCreate, + startedAt, + tc, + "Create iperf3 network", + map[string]any{ + "stdout": strings.TrimSpace(string(stdout)), + "stderr": strings.TrimSpace(string(stderr)), + }) + return result, commandError("create iperf3 network", err, stdout, stderr) + } + + // add some delay to ensure the network is ready before starting the server + time.Sleep(2 * time.Second) + + a.serverName = tc.Trial.ID + "-iperf3-server" + + args := []string{ + "run", + "-it", + "--name", a.serverName, + "--runtime", tc.Trial.RuntimeHandler, + "--network", a.networkName, + image(tc), "-s", "--one-off", + } + + log.Printf("Starting background server: nerdctl %s\n", strings.Join(args, " ")) + + // 2. Use context.Background() so the server isn't killed if the parent context cancels + cmd := exec.CommandContext(context.Background(), "nerdctl", args...) + + // temporary log file to capture the server's output + logFile, err := os.CreateTemp("", "iperf3-server-*.log") + if err != nil { + return stageResult( + harnessruntime.StageCreate, + startedAt, tc, + "Create iperf3 server log file", + nil, + ), + fmt.Errorf("failed to create log file for iperf3 server: %w", err) + } + defer logFile.Close() + + cmd.Stdin = os.Stdin + cmd.Stdout = logFile + cmd.Stderr = logFile + + // Detach from the parent's terminal/session. + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setsid: true, + } + + if err := cmd.Start(); err != nil { + result := stageResult(harnessruntime.StageCreate, startedAt, tc, "Start iperf3 server", nil) + // print error + log.Printf("Failed to start iperf3 server: %v\n", err) + + // clean up benchmark resources if the server fails to start + if cleanupErr := a.cleanupBenchmarkResources(ctx); cleanupErr != nil { + log.Printf("Failed to clean up benchmark resources: %v\n", cleanupErr) + } + + return result, fmt.Errorf("failed to start iperf3 server: %w", err) + } + + a.serverRunning = true + + go func() { _ = cmd.Wait() }() + + return stageResult( + harnessruntime.StageCreate, + startedAt, + tc, + "Start iperf3 server", + map[string]any{ + "server_name": a.serverName, + }, + ), nil +} + +func (a *Adapter) StartTask(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + log.Printf("Starting network benchmark trial=%s runtime=%s handler=%s image=%s\n", tc.Trial.ID, tc.Trial.RuntimeName, tc.Trial.RuntimeHandler, image(tc)) + startedAt := time.Now() + // wait for a few seconds to ensure the server is ready + time.Sleep(2 * time.Second) + + clientName := tc.Trial.ID + "-iperf3-client" + stdout, stderr, err := a.run( + ctx, + "run", + "-it", + "--rm", + "--name", clientName, + "--runtime", tc.Trial.RuntimeHandler, + "--network", a.networkName, + image(tc), + "-c", a.serverName, + "--json", + ) + + if err != nil { + log.Printf("Failed to run iperf3 client: %v\n", err) + // clean up benchmark resources if the client fails to run + if cleanupErr := a.cleanupBenchmarkResources(ctx); cleanupErr != nil { + log.Printf("Failed to clean up benchmark resources: %v\n", cleanupErr) + } + return stageResult( + harnessruntime.StageStart, + startedAt, + tc, + "Run iperf3 client", + map[string]any{ + "stdout": strings.TrimSpace(string(stdout)), + "stderr": strings.TrimSpace(string(stderr)), + }, + ), fmt.Errorf("failed to run iperf3 client: %w", err) + } + + jsonOutput, err := ExtractJSONObject(stdout) + if err != nil { + log.Printf("Failed to extract JSON from iperf3 client output: %v\n", err) + // clean up benchmark resources if the client fails to run + if cleanupErr := a.cleanupBenchmarkResources(ctx); cleanupErr != nil { + log.Printf("Failed to clean up benchmark resources: %v\n", cleanupErr) + } + + return stageResult( + harnessruntime.StageStart, + startedAt, + tc, + "Run iperf3 client", + map[string]any{ + "stdout": strings.TrimSpace(string(stdout)), + "stderr": strings.TrimSpace(string(stderr)), + }, + ), fmt.Errorf("failed to extract JSON from iperf3 client output: %w", err) + } + + var iperfResult map[string]any + + if err := json.Unmarshal(jsonOutput, &iperfResult); err != nil { + log.Printf("Failed to unmarshal JSON from iperf3 client output: %v\n", err) + // clean up benchmark resources if the client fails to run + if cleanupErr := a.cleanupBenchmarkResources(ctx); cleanupErr != nil { + log.Printf("Failed to clean up benchmark resources: %v\n", cleanupErr) + } + + return stageResult( + harnessruntime.StageStart, + startedAt, + tc, + "Run iperf3 client", + map[string]any{ + "stdout": strings.TrimSpace(string(stdout)), + "stderr": strings.TrimSpace(string(stderr)), + }, + ), fmt.Errorf("failed to unmarshal JSON from iperf3 client output: %w", err) + } + + return stageResult(harnessruntime.StageStart, startedAt, tc, "Run iperf3 client", map[string]any{ + "iperf3": iperfResult, + "stderr": strings.TrimSpace(string(stderr)), + }), nil +} + +func (a *Adapter) WaitReady(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + return completedStage(ctx, harnessruntime.StageWaitReady, "iperf3 client completed", tc) +} + +func (a *Adapter) Stop(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + log.Printf("Stopping network benchmark trial=%s runtime=%s handler=%s image=%s\n", tc.Trial.ID, tc.Trial.RuntimeName, tc.Trial.RuntimeHandler, image(tc)) + + startedAt := time.Now() + + err := a.removeServer(ctx) + + result := stageResult(harnessruntime.StageStop, startedAt, tc, "Remove iperf3 server", nil) + + if err != nil { + return result, err + } + return result, nil +} + +func (a *Adapter) DeleteTask(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + return completedStage(ctx, harnessruntime.StageDelete, "iperf3 containers removed", tc) +} + +func (a *Adapter) Cleanup( + ctx context.Context, + tc harnessruntime.TrialContext, +) (harnessruntime.StageResult, error) { + log.Printf("Cleaning up network benchmark trial=%s runtime=%s handler=%s image=%s\n", tc.Trial.ID, tc.Trial.RuntimeName, tc.Trial.RuntimeHandler, image(tc)) + + startedAt := time.Now() + + result := stageResult(harnessruntime.StageCleanup, startedAt, tc, "Cleanup benchmark resources", nil) + if err := a.cleanupBenchmarkResources(ctx); err != nil { + return result, err + } + + return result, nil +} + +func (a *Adapter) removeServer(ctx context.Context) error { + if !a.serverRunning { + return nil + } + stdout, stderr, err := a.run(ctx, "rm", "-f", a.serverName) + if err != nil { + return commandError("remove iperf3 server", err, stdout, stderr) + } + a.serverRunning = false + return nil +} + +func (a *Adapter) cleanupBenchmarkResources(ctx context.Context) error { + log.Printf("Cleaning up network benchmark resources: server=%s network=%s\n", a.serverName, a.networkName) + + errors := []error{} + if err := a.removeServer(ctx); err != nil { + errors = append(errors, err) + } + + if a.networkName != "" { + stdout, stderr, err := a.run(ctx, "network", "rm", a.networkName) + if err != nil { + errors = append(errors, commandError("remove iperf3 network", err, stdout, stderr)) + } + } + + if len(errors) > 0 { + return fmt.Errorf("cleanup errors: %v", errors) + } + + return nil +} + +func image(tc harnessruntime.TrialContext) string { + if tc.Trial.Image == "" { + return defaultImage + } + return tc.Trial.Image +} + +func runNerdctl(ctx context.Context, args ...string) ([]byte, []byte, error) { + cmd := exec.CommandContext(ctx, "nerdctl", args...) + var stdout, stderr bytes.Buffer + cmd.Stdin = os.Stdin + cmd.Stdout = &stdout + cmd.Stderr = &stderr + // print the command being run for debugging purposes + log.Printf("Running command: nerdctl %s\n", strings.Join(args, " ")) + err := cmd.Run() + return stdout.Bytes(), stderr.Bytes(), err +} + +func commandError(action string, err error, stdout, stderr []byte) error { + return fmt.Errorf("%s: %w; stdout=%q; stderr=%q", action, err, stdout, stderr) +} + +func completedStage(ctx context.Context, stage harnessruntime.Stage, description string, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + startedAt := time.Now() + if err := ctx.Err(); err != nil { + return harnessruntime.StageResult{}, err + } + return stageResult(stage, startedAt, tc, description, nil), nil +} + +func stageResult(stage harnessruntime.Stage, startedAt time.Time, tc harnessruntime.TrialContext, description string, data map[string]any) harnessruntime.StageResult { + finishedAt := time.Now() + return harnessruntime.StageResult{ + Stage: stage, + StartedAt: startedAt, + FinishedAt: finishedAt, + Duration: finishedAt.Sub(startedAt), + Description: fmt.Sprintf("%s: trial=%s runtime=%s handler=%s image=%s", description, tc.Trial.ID, tc.Trial.RuntimeName, tc.Trial.RuntimeHandler, image(tc)), + Data: data, + } +} + +// ExtractJSONObject finds and returns the first valid JSON object embedded +// anywhere in noisy command output. +// +// It handles output containing: +// - ANSI terminal escape sequences +// - warnings before the JSON +// - SeaBIOS/iPXE boot output +// - logs or kernel messages after the JSON +func ExtractJSONObject(output []byte) ([]byte, error) { + for offset := 0; offset < len(output); { + relativeStart := bytes.IndexByte(output[offset:], '{') + if relativeStart == -1 { + break + } + + start := offset + relativeStart + candidate := output[start:] + + decoder := json.NewDecoder(bytes.NewReader(candidate)) + + var value json.RawMessage + if err := decoder.Decode(&value); err == nil { + // Ensure the extracted JSON is an object rather than an array, + // string, number, boolean, or null. + trimmed := bytes.TrimSpace(value) + if len(trimmed) > 0 && trimmed[0] == '{' { + result := make([]byte, len(trimmed)) + copy(result, trimmed) + return result, nil + } + } + + // This opening brace was not the beginning of valid JSON. + // Continue searching from the following byte. + offset = start + 1 + } + + return nil, fmt.Errorf("no valid JSON object found in output") +}