diff --git a/pkg/config/env.go b/pkg/config/env.go index 50fa53c25..bce1752ac 100644 --- a/pkg/config/env.go +++ b/pkg/config/env.go @@ -25,6 +25,10 @@ const ( // EnvDebug enables debug logging. EnvDebug = "DEVSY_DEBUG" + // EnvInsecureTLS, when truthy, disables TLS certificate verification for the + // shared HTTP client. Enabled by default. + EnvInsecureTLS = "DEVSY_INSECURE_TLS" + // EnvDisableTelemetry disables telemetry collection. EnvDisableTelemetry = "DEVSY_DISABLE_TELEMETRY" diff --git a/pkg/devcontainer/feature/features.go b/pkg/devcontainer/feature/features.go index 92c6fec71..1b4070dfd 100644 --- a/pkg/devcontainer/feature/features.go +++ b/pkg/devcontainer/feature/features.go @@ -1,16 +1,15 @@ package feature import ( + "context" "encoding/json" "fmt" "io" - "net/http" "os" "path" "path/filepath" "regexp" "strings" - "time" pkgconfig "github.com/devsy-org/devsy/pkg/config" "github.com/devsy-org/devsy/pkg/devcontainer/config" @@ -546,66 +545,13 @@ func downloadFeatureFromURL( ) error { log.Debugf("starting feature download: url=%s, destFile=%s", url, destFile) - // #nosec G301 -- TODO Consider using a more secure permission setting and ownership if needed. - err := os.MkdirAll(filepath.Dir(destFile), 0o755) - if err != nil { - return fmt.Errorf("create feature folder: %w", err) - } - - attempt := 0 - for range 3 { - if attempt > 0 { - delay := time.Duration(1<= 400 { - return fmt.Errorf("GET request failed, status code is %d", resp.StatusCode) - } - - file, err := os.Create(filepath.Clean(destFile)) //nolint:gosec // path from internal resolution - if err != nil { - return fmt.Errorf("create download file: %w", err) - } - defer func() { _ = file.Close() }() - - _, err = io.Copy(file, resp.Body) - if err != nil { - return fmt.Errorf("download feature: %w", err) + if err := devsyhttp.DownloadToFile( + context.Background(), url, destFile, devsyhttp.WithHeaders(httpHeaders), + ); err != nil { + return err } + log.Infof("Feature download completed successfully: url=%s, destFile=%s", url, destFile) return nil } diff --git a/pkg/http/download.go b/pkg/http/download.go new file mode 100644 index 000000000..1ab43b501 --- /dev/null +++ b/pkg/http/download.go @@ -0,0 +1,267 @@ +package http + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sync" + "time" + + "github.com/devsy-org/devsy/pkg/log" + "k8s.io/apimachinery/pkg/util/wait" +) + +var DefaultDownloadBackoff = wait.Backoff{ + Duration: 1 * time.Second, + Factor: 2.0, + Steps: 3, +} + +const DefaultStallTimeout = 60 * time.Second + +var ( + downloadClientOnce sync.Once + downloadClient *http.Client +) + +func defaultDownloadClient() *http.Client { + downloadClientOnce.Do(func() { downloadClient = newStallClient(DefaultStallTimeout) }) + return downloadClient +} + +// newStallClient builds a download client with a stall timeout but no retry: +// DownloadToFile owns download retries (including truncation). +func newStallClient(timeout time.Duration) *http.Client { + rt := WithUserAgent(baseRoundTripper(), UserAgent()) + rt = NewStallTransport(rt, timeout) + return &http.Client{Transport: rt} +} + +type downloadConfig struct { + headers map[string]string + backoff wait.Backoff + client *http.Client + mode os.FileMode + wrap func(r io.Reader, totalSize int64) io.Reader + skipIfSized bool +} + +// DownloadOption customizes DownloadToFile. +type DownloadOption func(*downloadConfig) + +// WithHeaders sets request headers sent on every download attempt. +func WithHeaders(headers map[string]string) DownloadOption { + return func(c *downloadConfig) { c.headers = headers } +} + +// WithBackoff overrides the retry backoff policy. +func WithBackoff(backoff wait.Backoff) DownloadOption { + return func(c *downloadConfig) { c.backoff = backoff } +} + +// WithClient overrides the HTTP client used for the download. +func WithClient(client *http.Client) DownloadOption { + return func(c *downloadConfig) { c.client = client } +} + +// WithMode sets the permission bits for the destination file (default 0o600), +// for downloads that must be executable. +func WithMode(mode os.FileMode) DownloadOption { + return func(c *downloadConfig) { c.mode = mode } +} + +// WithProgress wraps the response body on each attempt, e.g. to report download +// progress. wrap receives the body reader and the advertised size (-1 when +// unknown) and returns the reader to copy from. +func WithProgress(wrap func(r io.Reader, totalSize int64) io.Reader) DownloadOption { + return func(c *downloadConfig) { c.wrap = wrap } +} + +// SkipIfSameSize skips the download when destPath already exists and its size +// matches the advertised Content-Length, reusing the cached file. +func SkipIfSameSize() DownloadOption { + return func(c *downloadConfig) { c.skipIfSized = true } +} + +// WithStallTimeout swaps in a client with the given inactivity timeout. +func WithStallTimeout(d time.Duration) DownloadOption { + return func(c *downloadConfig) { c.client = newStallClient(d) } +} + +// permanentError marks a failure that retrying will not resolve. +type permanentError struct{ err error } + +func (e *permanentError) Error() string { return e.err.Error() } +func (e *permanentError) Unwrap() error { return e.err } + +// DownloadToFile downloads url into destPath, retrying transient failures +// (connection errors, 5xx, 429, truncated transfers) and failing fast on 4xx. +func DownloadToFile(ctx context.Context, url, destPath string, opts ...DownloadOption) error { + cfg := downloadConfig{ + backoff: DefaultDownloadBackoff, + client: defaultDownloadClient(), + mode: 0o600, + } + for _, opt := range opts { + opt(&cfg) + } + + var lastErr error + err := wait.ExponentialBackoffWithContext( + ctx, + cfg.backoff, + func(ctx context.Context) (bool, error) { + lastErr = downloadOnce(ctx, &cfg, url, destPath) + switch { + case lastErr == nil: + return true, nil + case isPermanent(lastErr): + return false, lastErr + default: + log.Debugf("download attempt failed, retrying: url=%s err=%v", url, lastErr) + return false, nil + } + }, + ) + if wait.Interrupted(err) { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return lastErr + } + return err +} + +func isPermanent(err error) bool { + var perr *permanentError + return errors.As(err, &perr) +} + +type downloadAttempt struct { + cfg *downloadConfig + url string + destPath string +} + +func downloadOnce(ctx context.Context, cfg *downloadConfig, url, destPath string) error { + a := &downloadAttempt{cfg: cfg, url: url, destPath: destPath} + return a.do(ctx) +} + +func (a *downloadAttempt) do(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, a.url, nil) + if err != nil { + return &permanentError{fmt.Errorf("download %s: %w", a.url, err)} + } + for key, value := range a.cfg.headers { + req.Header.Set(key, value) + } + + resp, err := a.cfg.client.Do(req) + if err != nil { + return fmt.Errorf("download %s: %w", a.url, err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode >= http.StatusBadRequest { + statusErr := fmt.Errorf("download %s: %s", a.url, resp.Status) + if retryableStatus(resp.StatusCode) { + return statusErr + } + return &permanentError{statusErr} + } + + return a.copyToFile(resp) +} + +func retryableStatus(code int) bool { + return code >= http.StatusInternalServerError || code == http.StatusTooManyRequests +} + +func (a *downloadAttempt) copyToFile(resp *http.Response) error { + if alreadyDownloaded(a.cfg, resp, a.destPath) { + return nil + } + + destPath := filepath.Clean(a.destPath) + destDir := filepath.Dir(destPath) + // #nosec G301 -- match existing download destinations; contents are public artifacts. + if err := os.MkdirAll(destDir, 0o755); err != nil { + return &permanentError{fmt.Errorf("create download folder: %w", err)} + } + + // Write to a temp file and rename on success so a failed transfer never + // leaves a partial file at destPath. + tmpPath, err := a.streamToTempFile(resp, destDir, destPath) + if err != nil { + return err + } + if err := os.Rename(tmpPath, destPath); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("download %s: %w", a.url, err) + } + return nil +} + +func (a *downloadAttempt) streamToTempFile( + resp *http.Response, destDir, destPath string, +) (string, error) { + tmp, err := os.CreateTemp(destDir, "."+filepath.Base(destPath)+".*.tmp") + if err != nil { + return "", &permanentError{fmt.Errorf("create download file: %w", err)} + } + tmpPath := tmp.Name() + committed := false + defer func() { + _ = tmp.Close() + if !committed { + _ = os.Remove(tmpPath) + } + }() + + if err := tmp.Chmod(a.cfg.mode); err != nil { + return "", &permanentError{fmt.Errorf("set download file mode: %w", err)} + } + + written, err := io.Copy(tmp, a.source(resp)) + if err != nil { + return "", fmt.Errorf("download %s: %w", a.url, err) + } + if err := verifyContentLength(resp, written, a.url); err != nil { + return "", err + } + if err := tmp.Close(); err != nil { + return "", fmt.Errorf("download %s: %w", a.url, err) + } + committed = true + return tmpPath, nil +} + +func (a *downloadAttempt) source(resp *http.Response) io.Reader { + if a.cfg.wrap != nil { + return a.cfg.wrap(resp.Body, resp.ContentLength) + } + return resp.Body +} + +func alreadyDownloaded(cfg *downloadConfig, resp *http.Response, destPath string) bool { + if !cfg.skipIfSized || resp.ContentLength < 0 { + return false + } + info, err := os.Stat(filepath.Clean(destPath)) + return err == nil && info.Size() == resp.ContentLength +} + +func verifyContentLength(resp *http.Response, written int64, url string) error { + if resp.ContentLength >= 0 && written != resp.ContentLength { + return fmt.Errorf( + "download %s: incomplete transfer, got %d of %d bytes", + url, written, resp.ContentLength, + ) + } + return nil +} diff --git a/pkg/http/download_test.go b/pkg/http/download_test.go new file mode 100644 index 000000000..52eb9f1e4 --- /dev/null +++ b/pkg/http/download_test.go @@ -0,0 +1,318 @@ +package http + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" +) + +func fastBackoff() DownloadOption { + return WithBackoff(wait.Backoff{Duration: time.Millisecond, Factor: 1.0, Steps: 3}) +} + +func TestDownloadToFileSuccess(t *testing.T) { + const body = "hello-payload" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(body)) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "out.bin") + if err := DownloadToFile(context.Background(), srv.URL, dest, fastBackoff()); err != nil { + t.Fatalf("DownloadToFile: %v", err) + } + got, err := os.ReadFile(dest) // #nosec G304 -- test-controlled temp path + if err != nil { + t.Fatal(err) + } + if string(got) != body { + t.Fatalf("content mismatch: got %q want %q", got, body) + } +} + +func TestDownloadToFileSendsHeaders(t *testing.T) { + var gotHeader string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotHeader = r.Header.Get("Foo-Header") + _, _ = w.Write([]byte("ok")) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "out.bin") + err := DownloadToFile(context.Background(), srv.URL, dest, + WithHeaders(map[string]string{"Foo-Header": "Bar"}), fastBackoff()) + if err != nil { + t.Fatalf("DownloadToFile: %v", err) + } + if gotHeader != "Bar" { + t.Fatalf("expected header to be sent, got %q", gotHeader) + } +} + +func TestDownloadToFileTruncatedRetriesThenReportsClearError(t *testing.T) { + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.Header().Set("Content-Length", "1000") + _, _ = w.Write([]byte("short")) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "out.bin") + err := DownloadToFile(context.Background(), srv.URL, dest, fastBackoff()) + if err == nil { + t.Fatal("expected error for repeated truncated transfers") + } + if !strings.Contains(err.Error(), "download") { + t.Fatalf("expected a download-attributed error, got %q", err) + } + if calls != 3 { + t.Fatalf("expected 3 attempts (transient), got %d", calls) + } +} + +func TestDownloadToFileLeavesNoPartialFileOnFailure(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "1000") + _, _ = w.Write([]byte("short")) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "out.bin") + if err := DownloadToFile(context.Background(), srv.URL, dest, fastBackoff()); err == nil { + t.Fatal("expected error for repeated truncated transfers") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatalf("expected no partial file at destination, stat err = %v", err) + } + entries, err := os.ReadDir(filepath.Dir(dest)) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("expected no leftover temp files, found %d entries", len(entries)) + } +} + +func TestDownloadToFilePreservesExistingFileOnFailure(t *testing.T) { + const existing = "previous-good-content" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "1000") + _, _ = w.Write([]byte("short")) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "out.bin") + if err := os.WriteFile(dest, []byte(existing), 0o600); err != nil { + t.Fatal(err) + } + + if err := DownloadToFile(context.Background(), srv.URL, dest, fastBackoff()); err == nil { + t.Fatal("expected error for repeated truncated transfers") + } + got, err := os.ReadFile(dest) // #nosec G304 -- test-controlled temp path + if err != nil { + t.Fatalf("expected existing file preserved, got %v", err) + } + if string(got) != existing { + t.Fatalf("existing file was clobbered, got %q", got) + } +} + +func TestDownloadToFileStallTimeoutAborts(t *testing.T) { + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "1000") + w.WriteHeader(http.StatusOK) + flusher, ok := w.(http.Flusher) + if !ok { + t.Error("expected flusher") + return + } + _, _ = w.Write([]byte("partial")) + flusher.Flush() + // Send headers and a little data, then hang: the stall watchdog must + // abort the attempt rather than block forever. + select { + case <-r.Context().Done(): + case <-release: + } + })) + defer srv.Close() + defer close(release) + + dest := filepath.Join(t.TempDir(), "out.bin") + err := DownloadToFile( + context.Background(), srv.URL, dest, + WithStallTimeout(50*time.Millisecond), fastBackoff(), + ) + if err == nil || !strings.Contains(err.Error(), "stalled") { + t.Fatalf("expected a stalled error, got %v", err) + } + if _, statErr := os.Stat(dest); !os.IsNotExist(statErr) { + t.Fatalf("expected no partial file after stall, stat err = %v", statErr) + } +} + +func TestDownloadToFileReportsContextCancellation(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + // Backoff longer than the context deadline, so cancellation happens during + // the wait after a transient (non-context) failure. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + slow := WithBackoff(wait.Backoff{Duration: 100 * time.Millisecond, Factor: 1, Steps: 3}) + + dest := filepath.Join(t.TempDir(), "out.bin") + err := DownloadToFile(ctx, srv.URL, dest, slow) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected context deadline error, got %v", err) + } +} + +func TestDownloadToFileRecoversAfterTransientFailure(t *testing.T) { + const body = "recovered-payload" + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + if calls == 1 { + w.Header().Set("Content-Length", "1000") + _, _ = w.Write([]byte("short")) + return + } + _, _ = w.Write([]byte(body)) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "out.bin") + if err := DownloadToFile(context.Background(), srv.URL, dest, fastBackoff()); err != nil { + t.Fatalf("expected recovery, got %v", err) + } + got, err := os.ReadFile(dest) // #nosec G304 -- test-controlled temp path + if err != nil { + t.Fatal(err) + } + if string(got) != body { + t.Fatalf("expected fresh content after retry, got %q", got) + } +} + +func TestDownloadToFilePermanentStatusFailsFast(t *testing.T) { + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "out.bin") + err := DownloadToFile(context.Background(), srv.URL, dest, fastBackoff()) + if err == nil || !strings.Contains(err.Error(), "404") { + t.Fatalf("expected a 404 error, got %v", err) + } + if calls != 1 { + t.Fatalf("expected 4xx to fail fast without retry, got %d attempts", calls) + } +} + +func TestDownloadToFileTooManyRequestsRetries(t *testing.T) { + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.WriteHeader(http.StatusTooManyRequests) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "out.bin") + err := DownloadToFile(context.Background(), srv.URL, dest, fastBackoff()) + if err == nil || !strings.Contains(err.Error(), "429") { + t.Fatalf("expected a 429 error, got %v", err) + } + if calls != 3 { + t.Fatalf("expected 429 to be retried, got %d attempts", calls) + } +} + +func TestDownloadToFileWithMode(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("binary")) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "bin") + if err := DownloadToFile( + context.Background(), + srv.URL, + dest, + WithMode(0o755), + fastBackoff(), + ); err != nil { + t.Fatalf("DownloadToFile: %v", err) + } + info, err := os.Stat(dest) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm()&0o100 == 0 { + t.Fatalf("expected owner-executable file, got mode %v", info.Mode().Perm()) + } +} + +func TestDownloadToFileSkipIfSameSize(t *testing.T) { + const body = "cached-bytes" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // Same length as the pre-existing file, but different content. + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + _, _ = io.WriteString(w, strings.Repeat("x", len(body))) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "out.bin") + if err := os.WriteFile(dest, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + if err := DownloadToFile( + context.Background(), srv.URL, dest, SkipIfSameSize(), fastBackoff(), + ); err != nil { + t.Fatalf("DownloadToFile: %v", err) + } + got, err := os.ReadFile(dest) // #nosec G304 -- test-controlled temp path + if err != nil { + t.Fatal(err) + } + if string(got) != body { + t.Fatalf("expected existing file preserved on size match, got %q", got) + } +} + +func TestDownloadToFileServerErrorRetries(t *testing.T) { + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "out.bin") + err := DownloadToFile(context.Background(), srv.URL, dest, fastBackoff()) + if err == nil { + t.Fatal("expected error for persistent 5xx") + } + if calls != 3 { + t.Fatalf("expected 5xx to be retried, got %d attempts", calls) + } +} diff --git a/pkg/http/http.go b/pkg/http/http.go index c3deda938..c665501fa 100644 --- a/pkg/http/http.go +++ b/pkg/http/http.go @@ -3,20 +3,59 @@ package http import ( "crypto/tls" "net/http" + "os" + "strconv" "sync" + + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/log" + "github.com/devsy-org/devsy/pkg/version" ) var ( - httpClient *http.Client + baseTransportOnce sync.Once + baseTransportInst *http.Transport + httpClientOnce sync.Once + httpClientInst *http.Client ) +// baseRoundTripper is the shared transport that purpose-built clients wrap. +func baseRoundTripper() *http.Transport { + baseTransportOnce.Do(func() { + t := http.DefaultTransport.(*http.Transport).Clone() + if insecureTLSEnabled() { + // #nosec G402 -- enabled with DEVSY_INSECURE_TLS env + t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + log.Warnf("TLS certificate verification is disabled (%s)", config.EnvInsecureTLS) + } + baseTransportInst = t + }) + return baseTransportInst +} + +// GetHTTPClient returns the shared client: base transport, stall guard, +// user-agent, and retry of idempotent requests. The stall guard bounds idle +// header/body waits, so it must not be used for intentionally-idle streams. func GetHTTPClient() *http.Client { httpClientOnce.Do(func() { - customTransport := http.DefaultTransport.(*http.Transport).Clone() - customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - httpClient = &http.Client{Transport: customTransport} + rt := NewStallTransport(baseRoundTripper(), DefaultStallTimeout) + rt = WithUserAgent(rt, UserAgent()) + rt = NewRetryTransport(rt, DefaultRetry) + httpClientInst = &http.Client{Transport: rt} }) + return httpClientInst +} + +// UserAgent is the value sent on requests made through the package's clients. +func UserAgent() string { + return config.BinaryName + "/" + version.GetVersion() +} - return httpClient +// insecureTLSEnabled reports whether TLS certificate verification should be +// disabled for the shared HTTP client. Verification is on by default and is +// only disabled when EnvInsecureTLS is set to a truthy value. +func insecureTLSEnabled() bool { + enabled, _ := strconv.ParseBool(os.Getenv(config.EnvInsecureTLS)) + return enabled } diff --git a/pkg/http/http_test.go b/pkg/http/http_test.go new file mode 100644 index 000000000..54d17ca20 --- /dev/null +++ b/pkg/http/http_test.go @@ -0,0 +1,30 @@ +package http + +import ( + "testing" + + "github.com/devsy-org/devsy/pkg/config" +) + +func TestInsecureTLSEnabled(t *testing.T) { + tests := []struct { + name string + value string + want bool + }{ + {name: "unset defaults to secure", value: "", want: false}, + {name: "true enables insecure", value: "true", want: true}, + {name: "1 enables insecure", value: "1", want: true}, + {name: "false stays secure", value: "false", want: false}, + {name: "garbage stays secure", value: "yes-please", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(config.EnvInsecureTLS, tt.value) + if got := insecureTLSEnabled(); got != tt.want { + t.Errorf("insecureTLSEnabled() with %q = %v, want %v", tt.value, got, tt.want) + } + }) + } +} diff --git a/pkg/http/retry.go b/pkg/http/retry.go new file mode 100644 index 000000000..1e13f6e63 --- /dev/null +++ b/pkg/http/retry.go @@ -0,0 +1,136 @@ +package http + +import ( + "context" + "errors" + "io" + "net/http" + "strconv" + "time" + + "github.com/devsy-org/devsy/pkg/log" +) + +type RetryConfig struct { + MaxAttempts int // <=1 disables retry + BaseDelay time.Duration + MaxDelay time.Duration +} + +var DefaultRetry = RetryConfig{MaxAttempts: 3, BaseDelay: time.Second, MaxDelay: 30 * time.Second} + +const maxDrainOnRetry = 4 << 10 + +// RetryTransport retries idempotent requests on connection errors, 5xx and 429, +// honoring Retry-After. Non-idempotent and cancelled requests pass through. +type RetryTransport struct { + base http.RoundTripper + cfg RetryConfig +} + +func NewRetryTransport(base http.RoundTripper, cfg RetryConfig) http.RoundTripper { + if base == nil { + base = http.DefaultTransport + } + if cfg.MaxAttempts <= 1 { + return base + } + return &RetryTransport{base: base, cfg: cfg} +} + +func (t *RetryTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if !retriableRequest(req) { + return t.base.RoundTrip(req) + } + + for attempt := 1; ; attempt++ { + resp, err := t.base.RoundTrip(req) + if attempt >= t.cfg.MaxAttempts || !t.retryable(resp, err) { + return resp, err + } + + delay := t.backoff(attempt) + if resp != nil { + if after, ok := ParseRetryAfter(resp); ok { + delay = minDuration(after, t.cfg.MaxDelay) + } + drainAndClose(resp) + } + if sleepErr := sleepContext(req.Context(), delay); sleepErr != nil { + return nil, sleepErr + } + log.Debugf("retrying request: method=%s url=%s attempt=%d", req.Method, req.URL, attempt+1) + } +} + +func (t *RetryTransport) retryable(resp *http.Response, err error) bool { + if err != nil { + return !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) + } + return resp.StatusCode >= http.StatusInternalServerError || + resp.StatusCode == http.StatusTooManyRequests +} + +func (t *RetryTransport) backoff(attempt int) time.Duration { + delay := t.cfg.BaseDelay << (attempt - 1) + if delay <= 0 { // shift overflow + return t.cfg.MaxDelay + } + return minDuration(delay, t.cfg.MaxDelay) +} + +// retriableRequest allows retry only for idempotent, bodyless requests; a +// consumed request body cannot be safely replayed. +func retriableRequest(req *http.Request) bool { + if req.Body != nil && req.Body != http.NoBody { + return false + } + return req.Method == "" || req.Method == http.MethodGet || req.Method == http.MethodHead +} + +// ParseRetryAfter reads a Retry-After header in delta-seconds or HTTP-date form. +func ParseRetryAfter(resp *http.Response) (time.Duration, bool) { + v := resp.Header.Get("Retry-After") + if v == "" { + return 0, false + } + if secs, err := strconv.Atoi(v); err == nil { + if secs < 0 { + return 0, false + } + return time.Duration(secs) * time.Second, true + } + if at, err := http.ParseTime(v); err == nil { + if d := time.Until(at); d > 0 { + return d, true + } + return 0, true + } + return 0, false +} + +func drainAndClose(resp *http.Response) { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxDrainOnRetry)) + _ = resp.Body.Close() +} + +func sleepContext(ctx context.Context, d time.Duration) error { + if d <= 0 { + return ctx.Err() + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func minDuration(a, b time.Duration) time.Duration { + if a < b { + return a + } + return b +} diff --git a/pkg/http/retry_test.go b/pkg/http/retry_test.go new file mode 100644 index 000000000..351fe45a2 --- /dev/null +++ b/pkg/http/retry_test.go @@ -0,0 +1,210 @@ +package http + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + "time" +) + +type stubRoundTripper struct { + calls int + fn func(attempt int) (*http.Response, error) +} + +func (s *stubRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + s.calls++ + return s.fn(s.calls) +} + +func stubResp(code int, hdr http.Header) *http.Response { + if hdr == nil { + hdr = http.Header{} + } + return &http.Response{ + StatusCode: code, + Header: hdr, + Body: io.NopCloser(strings.NewReader("body")), + } +} + +func fastRetry() RetryConfig { + return RetryConfig{MaxAttempts: 3, BaseDelay: time.Millisecond, MaxDelay: time.Millisecond} +} + +func mustGet(t *testing.T, method string) *http.Request { + t.Helper() + req, err := http.NewRequest(method, "http://example.test", nil) + if err != nil { + t.Fatal(err) + } + return req +} + +func TestRetryTransportRetriesServerErrorThenSucceeds(t *testing.T) { + stub := &stubRoundTripper{fn: func(attempt int) (*http.Response, error) { + if attempt < 3 { + return stubResp(http.StatusServiceUnavailable, nil), nil + } + return stubResp(http.StatusOK, nil), nil + }} + rt := NewRetryTransport(stub, fastRetry()) + + resp, err := rt.RoundTrip(mustGet(t, http.MethodGet)) + if err != nil { + t.Fatalf("RoundTrip: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + if stub.calls != 3 { + t.Fatalf("expected 3 attempts, got %d", stub.calls) + } +} + +func TestRetryTransportGivesUpAfterMaxAttempts(t *testing.T) { + stub := &stubRoundTripper{fn: func(int) (*http.Response, error) { + return stubResp(http.StatusBadGateway, nil), nil + }} + rt := NewRetryTransport(stub, fastRetry()) + + resp, err := rt.RoundTrip(mustGet(t, http.MethodGet)) + if err != nil { + t.Fatalf("RoundTrip: %v", err) + } + if resp.StatusCode != http.StatusBadGateway { + t.Fatalf("expected last 502 returned, got %d", resp.StatusCode) + } + if stub.calls != 3 { + t.Fatalf("expected 3 attempts, got %d", stub.calls) + } +} + +func TestRetryTransportDoesNotRetryNonIdempotent(t *testing.T) { + stub := &stubRoundTripper{fn: func(int) (*http.Response, error) { + return stubResp(http.StatusServiceUnavailable, nil), nil + }} + rt := NewRetryTransport(stub, fastRetry()) + + if _, err := rt.RoundTrip(mustGet(t, http.MethodPost)); err != nil { + t.Fatalf("RoundTrip: %v", err) + } + if stub.calls != 1 { + t.Fatalf("expected POST not retried, got %d attempts", stub.calls) + } +} + +func TestRetryTransportDoesNotRetryRequestWithBody(t *testing.T) { + stub := &stubRoundTripper{fn: func(int) (*http.Response, error) { + return stubResp(http.StatusServiceUnavailable, nil), nil + }} + rt := NewRetryTransport(stub, fastRetry()) + + req, err := http.NewRequest(http.MethodGet, "http://example.test", strings.NewReader("x")) + if err != nil { + t.Fatal(err) + } + req.GetBody = nil + if _, err := rt.RoundTrip(req); err != nil { + t.Fatalf("RoundTrip: %v", err) + } + if stub.calls != 1 { + t.Fatalf("expected no retry for a request with a body, got %d attempts", stub.calls) + } +} + +func TestRetryTransportDoesNotRetryCancelledContext(t *testing.T) { + stub := &stubRoundTripper{fn: func(int) (*http.Response, error) { + return nil, context.Canceled + }} + rt := NewRetryTransport(stub, fastRetry()) + + _, err := rt.RoundTrip(mustGet(t, http.MethodGet)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } + if stub.calls != 1 { + t.Fatalf("expected no retry on cancellation, got %d attempts", stub.calls) + } +} + +func TestRetryTransportHonorsRetryAfterAndContext(t *testing.T) { + stub := &stubRoundTripper{fn: func(int) (*http.Response, error) { + return stubResp(http.StatusTooManyRequests, http.Header{"Retry-After": {"3600"}}), nil + }} + // Large MaxDelay so the honored Retry-After (1h) is the delay; a short + // context deadline proves the sleep is context-aware. + rt := NewRetryTransport( + stub, + RetryConfig{MaxAttempts: 3, BaseDelay: time.Second, MaxDelay: time.Hour}, + ) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + _, err := rt.RoundTrip(mustGet(t, http.MethodGet).WithContext(ctx)) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected deadline exceeded while honoring Retry-After, got %v", err) + } + if stub.calls != 1 { + t.Fatalf("expected to block on Retry-After, got %d attempts", stub.calls) + } +} + +func TestRetryTransportBackoffOverflowClampsToMax(t *testing.T) { + rt := &RetryTransport{ + cfg: RetryConfig{MaxAttempts: 100, BaseDelay: time.Second, MaxDelay: 30 * time.Second}, + } + if got := rt.backoff(64); got != rt.cfg.MaxDelay { + t.Fatalf("expected overflowed backoff to clamp to MaxDelay, got %v", got) + } +} + +func TestNewRetryTransportDisabledReturnsBase(t *testing.T) { + base := http.DefaultTransport + if got := NewRetryTransport(base, RetryConfig{MaxAttempts: 1}); got != base { + t.Fatalf("expected base returned when retry disabled") + } +} + +func TestParseRetryAfter(t *testing.T) { + tests := []struct { + name string + value string + wantOK bool + want time.Duration + }{ + {"seconds", "5", true, 5 * time.Second}, + {"zero", "0", true, 0}, + {"negative", "-1", false, 0}, + {"garbage", "soon", false, 0}, + {"missing", "", false, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hdr := http.Header{} + if tt.value != "" { + hdr.Set("Retry-After", tt.value) + } + got, ok := ParseRetryAfter(&http.Response{Header: hdr}) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v", ok, tt.wantOK) + } + if ok && got != tt.want { + t.Fatalf("duration = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseRetryAfterHTTPDate(t *testing.T) { + future := time.Now().Add(2 * time.Hour).UTC().Format(http.TimeFormat) + hdr := http.Header{} + hdr.Set("Retry-After", future) + got, ok := ParseRetryAfter(&http.Response{Header: hdr}) + if !ok || got <= 0 { + t.Fatalf("expected a positive delay for a future date, got %v ok=%v", got, ok) + } +} diff --git a/pkg/http/stall.go b/pkg/http/stall.go new file mode 100644 index 000000000..0ec9f1df3 --- /dev/null +++ b/pkg/http/stall.go @@ -0,0 +1,110 @@ +package http + +import ( + "context" + "errors" + "io" + "net/http" + "sync" + "time" +) + +var ErrStalled = errors.New("stalled: no progress within timeout") + +// StallTransport aborts a request that makes no progress within timeout. +// Unsafe for intentionally-idle responses (long-poll, watch streams). +type StallTransport struct { + base http.RoundTripper + timeout time.Duration +} + +func NewStallTransport(base http.RoundTripper, timeout time.Duration) http.RoundTripper { + if base == nil { + base = http.DefaultTransport + } + if timeout <= 0 { + return base + } + return &StallTransport{base: base, timeout: timeout} +} + +func (t *StallTransport) RoundTrip(req *http.Request) (*http.Response, error) { + ctx, cancel := context.WithCancel(req.Context()) + wd := newStallWatchdog(t.timeout, cancel) + + resp, err := t.base.RoundTrip(req.WithContext(ctx)) + if err != nil { + wd.stop() + cancel() + if wd.fired() { + return nil, ErrStalled + } + return nil, err + } + + wd.reset() + resp.Body = &stallBody{rc: resp.Body, wd: wd, cancel: cancel} + return resp, nil +} + +type stallBody struct { + rc io.ReadCloser + wd *stallWatchdog + cancel context.CancelFunc +} + +func (b *stallBody) Read(p []byte) (int, error) { + n, err := b.rc.Read(p) + if n > 0 { + b.wd.reset() + } + if err != nil && !errors.Is(err, io.EOF) && b.wd.fired() { + return n, ErrStalled + } + return n, err +} + +func (b *stallBody) Close() error { + b.wd.stop() + b.cancel() + return b.rc.Close() +} + +type stallWatchdog struct { + timeout time.Duration + timer *time.Timer + mu sync.Mutex + triggered bool +} + +func newStallWatchdog(timeout time.Duration, cancel context.CancelFunc) *stallWatchdog { + w := &stallWatchdog{timeout: timeout} + w.timer = time.AfterFunc(timeout, func() { + w.mu.Lock() + w.triggered = true + w.mu.Unlock() + cancel() + }) + return w +} + +func (w *stallWatchdog) reset() { + w.mu.Lock() + defer w.mu.Unlock() + if w.triggered { + return + } + w.timer.Reset(w.timeout) +} + +func (w *stallWatchdog) stop() { + w.mu.Lock() + defer w.mu.Unlock() + w.timer.Stop() +} + +func (w *stallWatchdog) fired() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.triggered +} diff --git a/pkg/http/stall_test.go b/pkg/http/stall_test.go new file mode 100644 index 000000000..3747229c3 --- /dev/null +++ b/pkg/http/stall_test.go @@ -0,0 +1,114 @@ +package http + +import ( + "errors" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestNewStallTransportPassthroughWhenDisabled(t *testing.T) { + base := http.DefaultTransport + if got := NewStallTransport(base, 0); got != base { + t.Fatalf("expected base transport returned unchanged when timeout <= 0") + } + if _, ok := NewStallTransport(base, time.Second).(*StallTransport); !ok { + t.Fatalf("expected a *StallTransport when timeout > 0") + } +} + +func TestStallTransportAbortsIdleBody(t *testing.T) { + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + _, _ = w.Write([]byte("partial")) + w.(http.Flusher).Flush() + select { + case <-r.Context().Done(): + case <-release: + } + })) + defer srv.Close() + defer close(release) + + client := &http.Client{Transport: NewStallTransport(http.DefaultTransport, 50*time.Millisecond)} + resp, err := client.Get(srv.URL) + if err != nil { + t.Fatalf("Get: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + _, err = io.ReadAll(resp.Body) + if !errors.Is(err, ErrStalled) { + t.Fatalf("expected ErrStalled reading a stalled body, got %v", err) + } +} + +func TestStallTransportAllowsProgressingBody(t *testing.T) { + const body = "steadily-delivered-payload" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(body)) + })) + defer srv.Close() + + client := &http.Client{Transport: NewStallTransport(http.DefaultTransport, 50*time.Millisecond)} + resp, err := client.Get(srv.URL) + if err != nil { + t.Fatalf("Get: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + got, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("expected clean read, got %v", err) + } + if string(got) != body { + t.Fatalf("body mismatch: got %q want %q", got, body) + } +} + +func TestStallTransportFollowsRedirect(t *testing.T) { + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("final-content")) + })) + defer origin.Close() + redir := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, origin.URL, http.StatusFound) + })) + defer redir.Close() + + client := &http.Client{Transport: NewStallTransport(http.DefaultTransport, 5*time.Second)} + resp, err := client.Get(redir.URL) + if err != nil { + t.Fatalf("Get through redirect: %v", err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + if string(body) != "final-content" { + t.Fatalf("got %q", body) + } +} + +func TestStallTransportHeaderStall(t *testing.T) { + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-release: + } + })) + defer srv.Close() + defer close(release) + + client := &http.Client{Transport: NewStallTransport(http.DefaultTransport, 50*time.Millisecond)} + _, err := client.Get(srv.URL) + if !errors.Is(err, ErrStalled) { + t.Fatalf("expected ErrStalled when headers never arrive, got %v", err) + } +} diff --git a/pkg/http/useragent.go b/pkg/http/useragent.go new file mode 100644 index 000000000..c552c4705 --- /dev/null +++ b/pkg/http/useragent.go @@ -0,0 +1,28 @@ +package http + +import "net/http" + +// WithUserAgent wraps base so requests that don't already carry a User-Agent +// header get the given value. An empty agent returns base unchanged. +func WithUserAgent(base http.RoundTripper, agent string) http.RoundTripper { + if base == nil { + base = http.DefaultTransport + } + if agent == "" { + return base + } + return &userAgentTransport{base: base, agent: agent} +} + +type userAgentTransport struct { + base http.RoundTripper + agent string +} + +func (t *userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Header.Get("User-Agent") == "" { + req = req.Clone(req.Context()) + req.Header.Set("User-Agent", t.agent) + } + return t.base.RoundTrip(req) +} diff --git a/pkg/http/useragent_test.go b/pkg/http/useragent_test.go new file mode 100644 index 000000000..db7ac0169 --- /dev/null +++ b/pkg/http/useragent_test.go @@ -0,0 +1,50 @@ +package http + +import ( + "net/http" + "testing" +) + +type recordingRoundTripper struct{ lastUA string } + +func (r *recordingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + r.lastUA = req.Header.Get("User-Agent") + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil +} + +func TestWithUserAgentSetsHeaderWhenAbsent(t *testing.T) { + rec := &recordingRoundTripper{} + rt := WithUserAgent(rec, "devsy/test") + + req, _ := http.NewRequest(http.MethodGet, "http://example.test", nil) + if _, err := rt.RoundTrip(req); err != nil { + t.Fatal(err) + } + if rec.lastUA != "devsy/test" { + t.Fatalf("expected user-agent set, got %q", rec.lastUA) + } + if req.Header.Get("User-Agent") != "" { + t.Fatalf("original request must not be mutated, got %q", req.Header.Get("User-Agent")) + } +} + +func TestWithUserAgentPreservesExisting(t *testing.T) { + rec := &recordingRoundTripper{} + rt := WithUserAgent(rec, "devsy/test") + + req, _ := http.NewRequest(http.MethodGet, "http://example.test", nil) + req.Header.Set("User-Agent", "caller/1.0") + if _, err := rt.RoundTrip(req); err != nil { + t.Fatal(err) + } + if rec.lastUA != "caller/1.0" { + t.Fatalf("expected caller user-agent preserved, got %q", rec.lastUA) + } +} + +func TestWithUserAgentEmptyReturnsBase(t *testing.T) { + base := http.DefaultTransport + if got := WithUserAgent(base, ""); got != base { + t.Fatalf("expected base returned for empty agent") + } +} diff --git a/pkg/ide/codeserver/codeserver.go b/pkg/ide/codeserver/codeserver.go index 0f4070b74..e90955b39 100644 --- a/pkg/ide/codeserver/codeserver.go +++ b/pkg/ide/codeserver/codeserver.go @@ -1,8 +1,8 @@ package codeserver import ( + "context" "fmt" - "net/http" "os" "os/exec" "path/filepath" @@ -13,7 +13,6 @@ import ( "github.com/devsy-org/devsy/pkg/config" copy2 "github.com/devsy-org/devsy/pkg/copy" "github.com/devsy-org/devsy/pkg/extract" - devsyhttp "github.com/devsy-org/devsy/pkg/http" "github.com/devsy-org/devsy/pkg/ide" "github.com/devsy-org/devsy/pkg/ide/vscode" "github.com/devsy-org/devsy/pkg/log" @@ -121,7 +120,9 @@ func (c *CodeServer) Install() error { vscode.InstallAPKRequirements() - if err := downloadAndExtract(c.getReleaseURL(), location); err != nil { + if err := ide.DownloadAndExtract( + context.Background(), c.getReleaseURL(), location, extract.StripLevels(1), + ); err != nil { return err } @@ -271,28 +272,6 @@ func (c *CodeServer) installSettings() error { return nil } -// downloadAndExtract fetches the release tarball at url and extracts it under -// location. Cleans up a partial extraction on failure so retries start fresh. -func downloadAndExtract(url, location string) error { - resp, err := devsyhttp.GetHTTPClient().Get(url) // #nosec G107 -- URL comes from VersionOption. - if err != nil { - return err - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode >= http.StatusBadRequest { - return fmt.Errorf("download code-server: %s returned %s", url, resp.Status) - } - - if err := extract.Extract(resp.Body, location, extract.StripLevels(1)); err != nil { - if rmErr := os.RemoveAll(location); rmErr != nil { - log.Warnf("cleanup partial install: path=%s err=%v", location, rmErr) - } - return fmt.Errorf("extract code-server: %w", err) - } - return nil -} - // suOrSh builds an *exec.Cmd that runs runCommand either as the unprivileged // user (via `su -c`) or as the current user (via `sh -c`). All arg // elements are constants from this package — runCommand itself is built from diff --git a/pkg/ide/download.go b/pkg/ide/download.go new file mode 100644 index 000000000..32117abfa --- /dev/null +++ b/pkg/ide/download.go @@ -0,0 +1,134 @@ +package ide + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/devsy-org/devsy/pkg/extract" + devsyhttp "github.com/devsy-org/devsy/pkg/http" + "github.com/devsy-org/devsy/pkg/log" +) + +// DownloadAndExtract downloads the archive at url and extracts it into destDir. +// It stages into a sibling dir and swaps on success, so a failure leaves any +// pre-existing install intact. +func DownloadAndExtract( + ctx context.Context, + url, destDir string, + opts ...extract.Option, +) error { + tmpPath, err := downloadToTemp(ctx, url) + if err != nil { + return err + } + defer func() { _ = os.Remove(tmpPath) }() + + stagingDir, err := extractToStaging(url, tmpPath, destDir, opts...) + if err != nil { + return err + } + return swapIntoPlace(stagingDir, destDir) +} + +func downloadToTemp(ctx context.Context, url string) (string, error) { + tmpFile, err := os.CreateTemp("", "devsy-ide-*.tar.gz") + if err != nil { + return "", fmt.Errorf("create temp file: %w", err) + } + tmpPath := tmpFile.Name() + _ = tmpFile.Close() + + if err := devsyhttp.DownloadToFile(ctx, url, tmpPath); err != nil { + _ = os.Remove(tmpPath) + return "", err + } + return tmpPath, nil +} + +func extractToStaging( + url, tmpPath, destDir string, + opts ...extract.Option, +) (string, error) { + file, err := os.Open(filepath.Clean(tmpPath)) + if err != nil { + return "", err + } + defer func() { _ = file.Close() }() + + // #nosec G301 -- IDE install dir + if err := os.MkdirAll(filepath.Dir(destDir), 0o755); err != nil { + return "", fmt.Errorf("create install parent: %w", err) + } + stagingDir, err := os.MkdirTemp(filepath.Dir(destDir), "."+filepath.Base(destDir)+".staging.*") + if err != nil { + return "", fmt.Errorf("create staging dir: %w", err) + } + + if err := extract.Extract(file, stagingDir, opts...); err != nil { + cleanupStaging(stagingDir) + return "", fmt.Errorf("extract %s: %w", url, err) + } + + // MkdirTemp yields 0700; restore the 0755 install-dir mode. + // #nosec G302 -- IDE install dir + if err := os.Chmod(stagingDir, 0o755); err != nil { + cleanupStaging(stagingDir) + return "", fmt.Errorf("set install dir mode: %w", err) + } + return stagingDir, nil +} + +func cleanupStaging(stagingDir string) { + if rmErr := os.RemoveAll(stagingDir); rmErr != nil { + log.Warnf("cleanup staging dir: path=%s err=%v", stagingDir, rmErr) + } +} + +// swapIntoPlace replaces destDir with stagingDir, keeping the old contents +// until the swap succeeds. +func swapIntoPlace(stagingDir, destDir string) error { + backupDir := stagingDir + ".old" + hasBackup, err := stashExisting(destDir, backupDir) + if err != nil { + _ = os.RemoveAll(stagingDir) + return err + } + + if err := os.Rename(stagingDir, destDir); err != nil { + _ = os.RemoveAll(stagingDir) + if hasBackup { + if restoreErr := os.Rename(backupDir, destDir); restoreErr != nil { + return fmt.Errorf( + "install to %s failed (%w); restore failed (%v); prior install at %s", + destDir, err, restoreErr, backupDir, + ) + } + } + return fmt.Errorf("install to %s: %w", destDir, err) + } + + if hasBackup { + if rmErr := os.RemoveAll(backupDir); rmErr != nil { + log.Warnf("cleanup old install: path=%s err=%v", backupDir, rmErr) + } + } + return nil +} + +// stashExisting moves an existing destDir aside so it can be restored if the +// swap fails; it reports whether a backup was made. +func stashExisting(destDir, backupDir string) (bool, error) { + _, err := os.Stat(destDir) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("inspect install dir: %w", err) + } + if err := os.Rename(destDir, backupDir); err != nil { + return false, fmt.Errorf("stage existing install: %w", err) + } + return true, nil +} diff --git a/pkg/ide/download_test.go b/pkg/ide/download_test.go new file mode 100644 index 000000000..cace65d6b --- /dev/null +++ b/pkg/ide/download_test.go @@ -0,0 +1,143 @@ +package ide + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func buildTarGz(t *testing.T, name, body string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader( + &tar.Header{Name: name, Mode: 0o755, Size: int64(len(body))}, + ); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestDownloadAndExtractSuccess(t *testing.T) { + archive := buildTarGz(t, "code", "#!/bin/sh\necho code\n") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(archive) + })) + defer srv.Close() + + dest := t.TempDir() + if err := DownloadAndExtract(context.Background(), srv.URL, dest); err != nil { + t.Fatalf("DownloadAndExtract: %v", err) + } + if _, err := os.Stat(filepath.Join(dest, "code")); err != nil { + t.Fatalf("expected extracted file: %v", err) + } + info, err := os.Stat(dest) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o755 { + t.Fatalf("expected install dir mode 0755, got %v", info.Mode().Perm()) + } +} + +func TestDownloadAndExtractCorruptArchiveReportsExtractError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // A complete transfer of bytes that are not a valid archive: the + // download succeeds, so the failure must be attributed to extraction. + _, _ = w.Write([]byte("not-a-tarball")) + })) + defer srv.Close() + + err := DownloadAndExtract(context.Background(), srv.URL, t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "extract") { + t.Fatalf("expected an extract-attributed error, got %v", err) + } +} + +func TestDownloadAndExtractCleansUpOnExtractFailure(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("not-a-tarball")) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "install") + if err := DownloadAndExtract(context.Background(), srv.URL, dest); err == nil { + t.Fatal("expected extract failure") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatalf("expected dest removed after extract failure, stat err = %v", err) + } +} + +func TestDownloadAndExtractPreservesExistingInstallOnFailure(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("not-a-tarball")) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "install") + if err := os.MkdirAll(dest, 0o750); err != nil { + t.Fatal(err) + } + existing := filepath.Join(dest, "existing") + if err := os.WriteFile(existing, []byte("keep-me"), 0o600); err != nil { + t.Fatal(err) + } + + if err := DownloadAndExtract(context.Background(), srv.URL, dest); err == nil { + t.Fatal("expected extract failure") + } + + got, err := os.ReadFile(existing) // #nosec G304 -- test-controlled temp path + if err != nil { + t.Fatalf("expected pre-existing install to remain intact: %v", err) + } + if string(got) != "keep-me" { + t.Fatalf("pre-existing content changed, got %q", got) + } +} + +func TestDownloadAndExtractReplacesExistingInstallOnSuccess(t *testing.T) { + archive := buildTarGz(t, "code", "#!/bin/sh\necho code\n") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(archive) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "install") + if err := os.MkdirAll(dest, 0o750); err != nil { + t.Fatal(err) + } + stale := filepath.Join(dest, "stale") + if err := os.WriteFile(stale, []byte("old"), 0o600); err != nil { + t.Fatal(err) + } + + if err := DownloadAndExtract(context.Background(), srv.URL, dest); err != nil { + t.Fatalf("DownloadAndExtract: %v", err) + } + if _, err := os.Stat(filepath.Join(dest, "code")); err != nil { + t.Fatalf("expected extracted file: %v", err) + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatalf("expected stale file replaced, stat err = %v", err) + } +} diff --git a/pkg/ide/fleet/fleet.go b/pkg/ide/fleet/fleet.go index 950031952..b19cc8067 100644 --- a/pkg/ide/fleet/fleet.go +++ b/pkg/ide/fleet/fleet.go @@ -2,6 +2,7 @@ package fleet import ( "bytes" + "context" "fmt" "io" "os" @@ -83,30 +84,11 @@ func (o *FleetServer) Install(projectDir string) error { // download binary log.Infof("Downloading fleet") - resp, err := devsyhttp.GetHTTPClient().Get(url) - if err != nil { + if err := devsyhttp.DownloadToFile( + context.Background(), url, fleetBinary, devsyhttp.WithMode(0o755), + ); err != nil { return err } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != 200 { - return fmt.Errorf( - "unexpected status code while trying to download fleet from %s: %d", - url, - resp.StatusCode, - ) - } - - f, err := os.OpenFile(fleetBinary, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o755) // #nosec G302,G304 - if err != nil { - return err - } - defer func() { _ = f.Close() }() - - _, err = io.Copy(f, resp.Body) - if err != nil { - return fmt.Errorf("download fleet: %w", err) - } - _ = f.Close() // chown location if o.userName != "" { diff --git a/pkg/ide/jetbrains/generic.go b/pkg/ide/jetbrains/generic.go index 34b976125..3f84bffbe 100644 --- a/pkg/ide/jetbrains/generic.go +++ b/pkg/ide/jetbrains/generic.go @@ -1,9 +1,9 @@ package jetbrains import ( + "context" "fmt" "io" - "net/http" "net/url" "os" "os/exec" @@ -244,48 +244,15 @@ func (o *GenericJetBrainsServer) download(targetFolder string) (string, error) { o.options.ID, ) - resp, err := fetchArchive(downloadURL) - if err != nil { - return "", err - } - defer func() { _ = resp.Body.Close() }() - - return saveArchive(targetPath, resp) -} - -func fetchArchive(downloadURL string) (*http.Response, error) { - resp, err := devsyhttp.GetHTTPClient().Get(downloadURL) - if err != nil { - return nil, fmt.Errorf("download binary: %w", err) - } - - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - _ = resp.Body.Close() - return nil, fmt.Errorf("download binary returned status code %d: %w", resp.StatusCode, err) - } - - return resp, nil -} - -func saveArchive(targetPath string, resp *http.Response) (string, error) { - stat, err := os.Stat(targetPath) - if err == nil && stat.Size() == resp.ContentLength { - return targetPath, nil - } - - file, err := os.Create(targetPath) - if err != nil { + if err := devsyhttp.DownloadToFile( + context.Background(), downloadURL, targetPath, + devsyhttp.SkipIfSameSize(), + devsyhttp.WithProgress(func(r io.Reader, totalSize int64) io.Reader { + return &ide.ProgressReader{Reader: r, TotalSize: totalSize} + }), + ); err != nil { return "", err } - defer func() { _ = file.Close() }() - - _, err = io.Copy(file, &ide.ProgressReader{ - Reader: resp.Body, - TotalSize: resp.ContentLength, - }) - if err != nil { - return "", fmt.Errorf("download file: %w", err) - } return targetPath, nil } diff --git a/pkg/ide/openvscode/openvscode.go b/pkg/ide/openvscode/openvscode.go index d204aee02..d087be096 100644 --- a/pkg/ide/openvscode/openvscode.go +++ b/pkg/ide/openvscode/openvscode.go @@ -1,6 +1,7 @@ package openvscode import ( + "context" "fmt" "os" "os/exec" @@ -12,7 +13,6 @@ import ( "github.com/devsy-org/devsy/pkg/config" copy2 "github.com/devsy-org/devsy/pkg/copy" "github.com/devsy-org/devsy/pkg/extract" - devsyhttp "github.com/devsy-org/devsy/pkg/http" "github.com/devsy-org/devsy/pkg/ide" "github.com/devsy-org/devsy/pkg/ide/vscode" "github.com/devsy-org/devsy/pkg/log" @@ -117,35 +117,33 @@ func (o *OpenVSCodeServer) Install() error { vscode.InstallAPKRequirements() - // download tar - resp, err := devsyhttp.GetHTTPClient().Get(url) - if err != nil { + if err := ide.DownloadAndExtract( + context.Background(), url, location, extract.StripLevels(1), + ); err != nil { return err } - defer func() { _ = resp.Body.Close() }() - - err = extract.Extract(resp.Body, location, extract.StripLevels(1)) - if err != nil { - return fmt.Errorf("extract vscode: %w", err) - } - // chown location - if o.userName != "" { - err = copy2.ChownR(location, o.userName) - if err != nil { - return fmt.Errorf("chown: %w", err) - } + if err := chownIfNeeded(o.userName, location); err != nil { + return err } - // paste settings - err = o.installSettings() - if err != nil { + if err := o.installSettings(); err != nil { return fmt.Errorf("install settings: %w", err) } return nil } +func chownIfNeeded(userName, location string) error { + if userName == "" { + return nil + } + if err := copy2.ChownR(location, userName); err != nil { + return fmt.Errorf("chown: %w", err) + } + return nil +} + func (o *OpenVSCodeServer) getReleaseUrl() string { var url string version := Options.GetValue(o.values, VersionOption) diff --git a/pkg/ide/rstudio/rstudio.go b/pkg/ide/rstudio/rstudio.go index b27614647..4e4dee80c 100644 --- a/pkg/ide/rstudio/rstudio.go +++ b/pkg/ide/rstudio/rstudio.go @@ -2,11 +2,11 @@ package rstudio import ( "bytes" + "context" _ "embed" "encoding/json" "fmt" "io" - "net/http" "os" "os/exec" "path/filepath" @@ -215,63 +215,26 @@ func getDownloadURL(version, ubuntuCodename, architecture string) string { } func download(targetFolder, downloadURL string) (string, error) { - err := os.MkdirAll(targetFolder, os.ModePerm) - if err != nil { + // #nosec G301 -- IDE install dir + if err := os.MkdirAll(targetFolder, 0o755); err != nil { return "", err } targetPath := filepath.Join(filepath.ToSlash(targetFolder), "rstudio-server.deb") - resp, err := devsyhttp.GetHTTPClient().Get(downloadURL) - if err != nil { - return "", fmt.Errorf("download deb: %w", err) - } - defer func() { _ = resp.Body.Close() }() - - if err := validateResponseStatus(resp, downloadURL); err != nil { - return "", err - } - - stat, err := os.Stat(targetPath) - if err == nil && stat.Size() == resp.ContentLength { - return targetPath, nil - } - - if err := saveResponseToFile(targetPath, resp); err != nil { + if err := devsyhttp.DownloadToFile( + context.Background(), downloadURL, targetPath, + devsyhttp.SkipIfSameSize(), + devsyhttp.WithProgress(func(r io.Reader, totalSize int64) io.Reader { + return &ide.ProgressReader{Reader: r, TotalSize: totalSize} + }), + ); err != nil { return "", err } return targetPath, nil } -func validateResponseStatus(resp *http.Response, downloadURL string) error { - if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusCreated { - return nil - } - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("RStudio version doesn't exist: %s", downloadURL) //nolint:all - } - return fmt.Errorf("download binary returned status code %d", resp.StatusCode) -} - -func saveResponseToFile(targetPath string, resp *http.Response) error { - file, err := os.Create(targetPath) - if err != nil { - return err - } - defer func() { _ = file.Close() }() - - _, err = io.Copy(file, &ide.ProgressReader{ - Reader: resp.Body, - TotalSize: resp.ContentLength, - }) - if err != nil { - return fmt.Errorf("download file: %w", err) - } - - return nil -} - func installDeb(debPath string) error { log.Info("Installing deb") diff --git a/pkg/ide/vscodeweb/vscodeweb.go b/pkg/ide/vscodeweb/vscodeweb.go index cec708006..e3bde6fbd 100644 --- a/pkg/ide/vscodeweb/vscodeweb.go +++ b/pkg/ide/vscodeweb/vscodeweb.go @@ -1,8 +1,8 @@ package vscodeweb import ( + "context" "fmt" - "net/http" "os" "os/exec" "path/filepath" @@ -12,8 +12,6 @@ import ( "github.com/devsy-org/devsy/pkg/command" "github.com/devsy-org/devsy/pkg/config" copy2 "github.com/devsy-org/devsy/pkg/copy" - "github.com/devsy-org/devsy/pkg/extract" - devsyhttp "github.com/devsy-org/devsy/pkg/http" "github.com/devsy-org/devsy/pkg/ide" "github.com/devsy-org/devsy/pkg/ide/vscode" "github.com/devsy-org/devsy/pkg/log" @@ -126,7 +124,7 @@ func (v *VSCodeWeb) Install() error { if !v.isInstalled(location, releaseURL) { vscode.InstallAPKRequirements() - if err := downloadAndExtract(releaseURL, location); err != nil { + if err := ide.DownloadAndExtract(context.Background(), releaseURL, location); err != nil { return err } @@ -296,30 +294,6 @@ func (v *VSCodeWeb) installSettings() error { return nil } -// downloadAndExtract fetches the CLI tarball at url and extracts it under -// location. The VS Code CLI tarball holds a single `code` binary at its root, -// so no strip levels are applied. Cleans up a partial extraction on failure so -// retries start fresh. -func downloadAndExtract(url, location string) error { - resp, err := devsyhttp.GetHTTPClient().Get(url) // #nosec G107 -- URL comes from VersionOption. - if err != nil { - return err - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode >= http.StatusBadRequest { - return fmt.Errorf("download VS Code CLI: %s returned %s", url, resp.Status) - } - - if err := extract.Extract(resp.Body, location); err != nil { - if rmErr := os.RemoveAll(location); rmErr != nil { - log.Warnf("cleanup partial install: path=%s err=%v", location, rmErr) - } - return fmt.Errorf("extract VS Code CLI: %w", err) - } - return nil -} - // suOrSh builds an *exec.Cmd that runs runCommand either as the unprivileged // user (via `su -c`) or as the current user (via `sh -c`). All arg // elements are constants from this package — runCommand itself is built from