feat/mvm-execute

cmd/librecode/chat.go

@@ -58,6 +58,7 @@ func runChat(cmd *cobra.Command, options chatRunOptions) error {
 		authStorage := container.AuthService().Storage
 		extensionManager := container.ExtensionService().Manager
 		cfg := container.ConfigService().Get()
+		chatWorkflows := container.ChatWorkflowService()
 
 		cwd, err := assistant.DefaultCWD("")
 		if err != nil {
@@ -75,6 +76,7 @@ func runChat(cmd *cobra.Command, options chatRunOptions) error {
 			Extensions: extensionManager,
 			Resources:  &resources,
 			Runtime:    runtime,
+			Workflows:  chatWorkflows.Runs,
 			Settings:   databaseService.Documents,
 			Models:     modelRegistry,
 			Auth:       authStorage,

cmd/librecode/main.go

@@ -7,9 +7,23 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+
+	"github.com/omarluq/librecode/internal/executeworker"
 )
 
 func main() {
+	if len(os.Args) == 2 && os.Args[1] == "__execute-worker" {
+		if err := executeworker.Serve(os.Stdin, os.Stdout); err != nil {
+			if _, writeErr := fmt.Fprintln(os.Stderr, err); writeErr != nil {
+				os.Exit(1)
+			}
+
+			os.Exit(1)
+		}
+
+		os.Exit(0)
+	}
+
 	os.Exit(run())
 }
 

cmd/librecode/prompt.go

@@ -1,9 +1,12 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
+	"time"
 
 	"github.com/samber/oops"
 	"github.com/spf13/cobra"
@@ -13,12 +16,17 @@ import (
 	"github.com/omarluq/librecode/internal/limitio"
 )
 
-const promptStdinLimitBytes int64 = 1 << 20
+const (
+	promptStdinLimitBytes int64 = 1 << 20
+	promptMetricsMode           = 0o600
+)
 
 type promptRunOptions struct {
-	SessionID   string
-	SessionName string
-	Resume      bool
+	SessionID    string
+	SessionName  string
+	ToolStrategy string
+	MetricsJSON  string
+	Resume       bool
 }
 
 func newPromptCmd() *cobra.Command {
@@ -35,6 +43,9 @@ func newPromptCmd() *cobra.Command {
 
 	cmd.Flags().StringVar(&options.SessionID, "session", "", "session id to append to")
 	cmd.Flags().StringVar(&options.SessionName, "name", "", "create a named session")
+	cmd.Flags().StringVar(&options.ToolStrategy, "tool-strategy", string(assistant.ToolStrategyHybrid),
+		"tool strategy: hybrid or direct")
+	cmd.Flags().StringVar(&options.MetricsJSON, "metrics-json", "", "write prompt metrics as JSON")
 	cmd.Flags().BoolVar(&options.Resume, "resume", false, "resume the latest session for this working directory")
 
 	return cmd
@@ -56,6 +67,15 @@ func runPrompt(cmd *cobra.Command, args []string, options promptRunOptions) erro
 }
 
 func validatePromptRunOptions(options promptRunOptions) error {
+	strategy := assistant.ToolStrategy(options.ToolStrategy)
+	if strategy == "" {
+		strategy = assistant.ToolStrategyHybrid
+	}
+
+	if strategy != assistant.ToolStrategyHybrid && strategy != assistant.ToolStrategyDirect {
+		return fmt.Errorf("invalid --tool-strategy %q: use hybrid or direct", options.ToolStrategy)
+	}
+
 	if options.Resume && options.SessionID != "" {
 		return errors.New("--resume cannot be used with --session")
 	}
@@ -80,9 +100,27 @@ func runPromptWithContainer(
 		return cliError(err, cliResolveWorkingDirectory)
 	}
 
-	response, err := runtime.Prompt(cmd.Context(), buildPromptRequest(cwd, message, options))
-	if err != nil {
-		return cliError(err, "run prompt")
+	strategy := normalizedToolStrategy(options.ToolStrategy)
+	metrics := new(assistant.RunMetrics)
+	ctx := assistant.WithToolStrategy(cmd.Context(), strategy)
+	ctx = assistant.WithRunMetrics(ctx, metrics)
+	request := buildPromptRequest(cwd, message, options)
+	request.OnEvent = metrics.ObserveStreamEvent
+	started := time.Now()
+	response, promptErr := runtime.Prompt(ctx, request)
+	elapsed := time.Since(started)
+
+	snapshot := metrics.Snapshot()
+	measured := &promptMetrics{
+		Strategy: string(strategy), Error: errorText(promptErr),
+		ProviderRoundTrips: snapshot.ProviderRoundTrips, ElapsedMilliseconds: elapsed.Milliseconds(),
+		InputTokens: snapshot.InputTokens, OutputTokens: snapshot.OutputTokens,
+		ToolCalls: snapshot.ToolCalls, NestedToolCalls: snapshot.NestedToolCalls,
+		TraceComplete: snapshot.TraceComplete, Success: promptErr == nil,
+	}
+
+	if err := promptExecutionError(promptErr, writePromptMetrics(options.MetricsJSON, measured)); err != nil {
+		return err
 	}
 
 	if _, err := fmt.Fprintln(cmd.OutOrStdout(), response.Text); err != nil {
@@ -92,6 +130,67 @@ func runPromptWithContainer(
 	return nil
 }
 
+func promptExecutionError(promptErr, metricsErr error) error {
+	if promptErr == nil {
+		return metricsErr
+	}
+
+	promptErr = cliError(promptErr, "run prompt")
+	if metricsErr != nil {
+		return errors.Join(promptErr, metricsErr)
+	}
+
+	return promptErr
+}
+
+type promptMetrics struct {
+	Strategy            string `json:"strategy"`
+	Error               string `json:"error"`
+	ProviderRoundTrips  int    `json:"provider_round_trips"`
+	ElapsedMilliseconds int64  `json:"elapsed_ms"`
+	InputTokens         int    `json:"input_tokens"`
+	OutputTokens        int    `json:"output_tokens"`
+	ToolCalls           int    `json:"tool_calls"`
+	NestedToolCalls     int    `json:"nested_tool_calls"`
+	TraceComplete       bool   `json:"trace_complete"`
+	Success             bool   `json:"success"`
+}
+
+func normalizedToolStrategy(value string) assistant.ToolStrategy {
+	strategy := assistant.ToolStrategy(value)
+	if strategy == "" {
+		return assistant.ToolStrategyHybrid
+	}
+
+	return strategy
+}
+
+func errorText(err error) string {
+	if err == nil {
+		return ""
+	}
+
+	return err.Error()
+}
+
+func writePromptMetrics(path string, metrics *promptMetrics) error {
+	if strings.TrimSpace(path) == "" {
+		return nil
+	}
+
+	encoded, err := json.MarshalIndent(metrics, "", "  ")
+	if err != nil {
+		return oops.In("cli").Code("encode_prompt_metrics").Wrapf(err, "encode prompt metrics")
+	}
+
+	encoded = append(encoded, '\n')
+	if err := os.WriteFile(path, encoded, promptMetricsMode); err != nil {
+		return oops.In("cli").Code("write_prompt_metrics").Wrapf(err, "write prompt metrics")
+	}
+
+	return nil
+}
+
 func buildPromptRequest(cwd, message string, options promptRunOptions) *assistant.PromptRequest {
 	return &assistant.PromptRequest{
 		OnEvent:        nil,

cmd/librecode/prompt_internal_test.go

@@ -1,7 +1,10 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -23,30 +26,43 @@ func TestValidatePromptRunOptions(t *testing.T) {
 		options promptRunOptions
 	}{
 		{
-			options: promptRunOptions{SessionID: "", SessionName: "", Resume: false},
+			options: promptRunOptions{SessionID: "", SessionName: "", ToolStrategy: "", MetricsJSON: "", Resume: false},
 			name:    "default",
 			wantErr: "",
 		},
 		{
-			options: promptRunOptions{SessionID: testSessionID, SessionName: "", Resume: false},
+			options: promptRunOptions{
+				SessionID: testSessionID, SessionName: "", ToolStrategy: "", MetricsJSON: "", Resume: false,
+			},
 			name:    "session only",
 			wantErr: "",
 		},
 		{
-			options: promptRunOptions{SessionID: "", SessionName: "", Resume: true},
+			options: promptRunOptions{SessionID: "", SessionName: "", ToolStrategy: "", MetricsJSON: "", Resume: true},
 			name:    "resume only",
 			wantErr: "",
 		},
 		{
-			options: promptRunOptions{SessionID: testSessionID, SessionName: "", Resume: true},
+			options: promptRunOptions{
+				SessionID: testSessionID, SessionName: "", ToolStrategy: "", MetricsJSON: "", Resume: true,
+			},
 			name:    "resume with session",
 			wantErr: "--resume cannot be used with --session",
 		},
 		{
-			options: promptRunOptions{SessionID: "", SessionName: testSessionName, Resume: true},
+			options: promptRunOptions{
+				SessionID: "", SessionName: testSessionName, ToolStrategy: "hybrid", MetricsJSON: "", Resume: true,
+			},
 			name:    "resume with name",
 			wantErr: "--resume cannot be used with --name",
 		},
+		{
+			options: promptRunOptions{
+				SessionID: "", SessionName: "", ToolStrategy: "invalid", MetricsJSON: "", Resume: false,
+			},
+			name:    "invalid strategy",
+			wantErr: "invalid --tool-strategy",
+		},
 	}
 
 	for _, testCase := range tests {
@@ -165,9 +181,7 @@ func TestBuildPromptRequest(t *testing.T) {
 	t.Parallel()
 
 	request := buildPromptRequest("/workspace", "hello", promptRunOptions{
-		SessionID:   "session-1",
-		SessionName: testSessionName,
-		Resume:      true,
+		SessionID: "session-1", SessionName: testSessionName, ToolStrategy: "", MetricsJSON: "", Resume: true,
 	})
 
 	require.NotNil(t, request)
@@ -183,6 +197,32 @@ func TestBuildPromptRequest(t *testing.T) {
 	assert.False(t, request.HideUserPrompt)
 }
 
+func TestWritePromptMetrics(t *testing.T) {
+	t.Parallel()
+
+	path := filepath.Join(t.TempDir(), "metrics.json")
+	metrics := &promptMetrics{
+		Strategy: "direct", Error: "", ProviderRoundTrips: 2, ElapsedMilliseconds: 12,
+		InputTokens: 10, OutputTokens: 4, ToolCalls: 1, NestedToolCalls: 0,
+		TraceComplete: true, Success: true,
+	}
+	require.NoError(t, writePromptMetrics(path, metrics))
+
+	encoded, err := os.ReadFile(filepath.Clean(path))
+	require.NoError(t, err)
+
+	var decoded promptMetrics
+	require.NoError(t, json.Unmarshal(encoded, &decoded))
+	assert.Equal(t, *metrics, decoded)
+}
+
+func TestNormalizedToolStrategyDefaultsToHybrid(t *testing.T) {
+	t.Parallel()
+
+	assert.Equal(t, "hybrid", string(normalizedToolStrategy("")))
+	assert.Equal(t, "direct", string(normalizedToolStrategy("direct")))
+}
+
 func TestPromptMessageRejectsStdinAboveLimit(t *testing.T) {
 	t.Parallel()
 

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/gofrs/uuid/v5 v5.4.0
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/invopop/jsonschema v0.14.0
+	github.com/mvm-sh/mvm v0.5.0
 	github.com/odvcencio/gotreesitter v0.38.0
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
 	github.com/pressly/goose/v3 v3.27.2

go.sum

@@ -135,12 +135,10 @@ github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6B
 github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mvm-sh/mvm v0.5.0 h1:XWII2Y8RLEzvFMuBWSacg6KLUTUp2weBin4lUlFrV+Y=
+github.com/mvm-sh/mvm v0.5.0/go.mod h1:2h9+ibS1DzdkMRNjGVs+pkU0blZZXDrEIUpcI93p4XA=
 github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
 github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
-github.com/odvcencio/gotreesitter v0.36.0 h1:WN9pMlzIEOcDoeWDSSeNHC9zpPi7kcZ66sYmf7eknDY=
-github.com/odvcencio/gotreesitter v0.36.0/go.mod h1:hBVkghd0paaYAVwd2087vfwdeU984bQbMo9LvpE0moo=
-github.com/odvcencio/gotreesitter v0.37.0 h1:9LO82hxtUJKIzDSfqhCZggCYTaxdX4JwXqydbbQi+iE=
-github.com/odvcencio/gotreesitter v0.37.0/go.mod h1:hBVkghd0paaYAVwd2087vfwdeU984bQbMo9LvpE0moo=
 github.com/odvcencio/gotreesitter v0.38.0 h1:6UkjeVOeihIqSilBw3ezUH6UK9korS5TfCmhXF9sFHc=
 github.com/odvcencio/gotreesitter v0.38.0/go.mod h1:hBVkghd0paaYAVwd2087vfwdeU984bQbMo9LvpE0moo=
 github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s=
@@ -357,20 +355,20 @@ lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl
 lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20=
 modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
-modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c=
-modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc=
+modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
 modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI=
 modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g=
 modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
-modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws=
-modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE=
+modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
+modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
 modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
 modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
 modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
 modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
 modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
-modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc=
-modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
+modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
 modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
 modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
 modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
@@ -379,8 +377,6 @@ modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0=
 modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI=
 modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
 modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak=
-modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA=
-modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8=
 modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ=
 modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os=
 modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
@@ -399,8 +395,6 @@ modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
 modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
 modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
 modernc.org/sqlite v1.26.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU=
-modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M=
-modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s=
 modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog=
 modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw=
 modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=