Add content asset redirect gateway

api/content_asset.go

@@ -0,0 +1,109 @@
+package api
+
+import (
+	"net/url"
+	"slices"
+	"strings"
+
+	"api.audius.co/config"
+	"api.audius.co/rendezvous"
+	"github.com/gofiber/fiber/v2"
+	"go.uber.org/zap"
+)
+
+const contentAssetUpstreamHeader = "X-Audius-Upstream"
+
+func (app *ApiServer) contentAssetRedirect(c *fiber.Ctx) error {
+	cid := c.Params("cid")
+	asset := c.Params("asset")
+
+	if !isValidContentPathSegment(cid) || (asset != "" && !isValidContentPathSegment(asset)) {
+		return fiber.NewError(fiber.StatusBadRequest, "invalid content path")
+	}
+
+	target, upstream, ok := contentAssetTarget(cid, asset, string(c.Context().URI().QueryString()))
+	if !ok {
+		return fiber.NewError(fiber.StatusServiceUnavailable, "no content nodes available")
+	}
+
+	c.Locals("upstream", upstream)
+	c.Set(contentAssetUpstreamHeader, upstream)
+	c.Set(fiber.HeaderCacheControl, "public, max-age=300")
+	app.logger.Debug("content asset redirect",
+		zap.String("cid", cid),
+		zap.String("asset", asset),
+		zap.String("upstream", upstream),
+	)
+
+	return c.Redirect(target, fiber.StatusTemporaryRedirect)
+}
+
+func contentAssetTarget(cid string, asset string, rawQuery string) (string, string, bool) {
+	for _, endpoint := range contentAssetCandidates(cid) {
+		target, ok := contentAssetURL(endpoint, cid, asset, rawQuery)
+		if ok {
+			return target, endpoint, true
+		}
+	}
+	return "", "", false
+}
+
+func contentAssetCandidates(cid string) []string {
+	candidates := make([]string, 0, len(config.Cfg.StoreAllNodes)+4)
+	seen := map[string]bool{}
+
+	appendCandidate := func(endpoint string) {
+		endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/")
+		if endpoint == "" || seen[endpoint] || isBlockedContentEndpoint(endpoint) {
+			return
+		}
+		u, err := url.Parse(endpoint)
+		if err != nil || u.Scheme == "" || u.Host == "" {
+			return
+		}
+		seen[endpoint] = true
+		candidates = append(candidates, endpoint)
+	}
+
+	for _, endpoint := range config.Cfg.StoreAllNodes {
+		appendCandidate(endpoint)
+	}
+
+	first, rest := rendezvous.GlobalHasher.Select(cid)
+	appendCandidate(first)
+	for _, endpoint := range rest {
+		appendCandidate(endpoint)
+	}
+
+	return candidates
+}
+
+func contentAssetURL(endpoint string, cid string, asset string, rawQuery string) (string, bool) {
+	u, err := url.Parse(endpoint)
+	if err != nil || u.Scheme == "" || u.Host == "" {
+		return "", false
+	}
+
+	path := "/content/" + url.PathEscape(cid)
+	if asset != "" {
+		path += "/" + url.PathEscape(asset)
+	}
+	u.Path = path
+	u.RawPath = ""
+	u.RawQuery = rawQuery
+	u.Fragment = ""
+
+	return u.String(), true
+}
+
+func isBlockedContentEndpoint(endpoint string) bool {
+	return slices.Contains(config.Cfg.BlacklistedNodes, endpoint) ||
+		slices.Contains(config.Cfg.DeadNodes, endpoint)
+}
+
+func isValidContentPathSegment(segment string) bool {
+	if segment == "" || segment == "." || segment == ".." {
+		return false
+	}
+	return !strings.ContainsAny(segment, `/\`+"\x00")
+}

api/content_asset_test.go

@@ -0,0 +1,129 @@
+package api
+
+import (
+	"io"
+	"net/http/httptest"
+	"testing"
+
+	"api.audius.co/config"
+	"api.audius.co/rendezvous"
+	"github.com/gofiber/fiber/v2"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+)
+
+func withContentAssetConfig(t *testing.T, storeAll []string, blacklisted []string, dead []string, nodes []config.Node) {
+	t.Helper()
+
+	oldStoreAll := config.Cfg.StoreAllNodes
+	oldBlacklisted := config.Cfg.BlacklistedNodes
+	oldDead := config.Cfg.DeadNodes
+	oldHasher := rendezvous.GlobalHasher
+
+	config.Cfg.StoreAllNodes = storeAll
+	config.Cfg.BlacklistedNodes = blacklisted
+	config.Cfg.DeadNodes = dead
+	rendezvous.Refresh(nodes)
+
+	t.Cleanup(func() {
+		config.Cfg.StoreAllNodes = oldStoreAll
+		config.Cfg.BlacklistedNodes = oldBlacklisted
+		config.Cfg.DeadNodes = oldDead
+		rendezvous.GlobalHasher = oldHasher
+	})
+}
+
+func TestContentAssetRedirectUsesStoreAllNode(t *testing.T) {
+	app := contentAssetTestApp()
+	withContentAssetConfig(t,
+		[]string{"https://store-all.test", "https://blocked.test"},
+		[]string{"https://blocked.test"},
+		nil,
+		[]config.Node{{Endpoint: "https://validator.test"}},
+	)
+
+	req := httptest.NewRequest("GET", "/content/test-cid/150x150.jpg?cache=bust", nil)
+	res, err := app.Test(req, -1)
+	require.NoError(t, err)
+
+	require.Equal(t, 307, res.StatusCode)
+	require.Equal(t, "https://store-all.test/content/test-cid/150x150.jpg?cache=bust", res.Header.Get("Location"))
+	require.Equal(t, "https://store-all.test", res.Header.Get(contentAssetUpstreamHeader))
+}
+
+func TestContentAssetRedirectSupportsLegacyContentPath(t *testing.T) {
+	app := contentAssetTestApp()
+	withContentAssetConfig(t,
+		[]string{"https://store-all.test"},
+		nil,
+		nil,
+		nil,
+	)
+
+	req := httptest.NewRequest("GET", "/content/test-cid", nil)
+	res, err := app.Test(req, -1)
+	require.NoError(t, err)
+
+	require.Equal(t, 307, res.StatusCode)
+	require.Equal(t, "https://store-all.test/content/test-cid", res.Header.Get("Location"))
+}
+
+func TestContentAssetRedirectFallsBackToRendezvous(t *testing.T) {
+	app := contentAssetTestApp()
+	withContentAssetConfig(t,
+		nil,
+		nil,
+		nil,
+		[]config.Node{{Endpoint: "https://validator.test"}},
+	)
+
+	req := httptest.NewRequest("GET", "/content/test-cid/150x150.jpg", nil)
+	res, err := app.Test(req, -1)
+	require.NoError(t, err)
+
+	require.Equal(t, 307, res.StatusCode)
+	require.Equal(t, "https://validator.test/content/test-cid/150x150.jpg", res.Header.Get("Location"))
+}
+
+func TestContentAssetRedirectNoCandidates(t *testing.T) {
+	app := contentAssetTestApp()
+	withContentAssetConfig(t, nil, nil, nil, nil)
+
+	req := httptest.NewRequest("GET", "/content/test-cid/150x150.jpg", nil)
+	res, err := app.Test(req, -1)
+	require.NoError(t, err)
+
+	require.Equal(t, 503, res.StatusCode)
+}
+
+func TestContentVerboseKeepsNodeListRoute(t *testing.T) {
+	app := contentAssetTestApp()
+	withContentAssetConfig(t,
+		[]string{"https://store-all.test"},
+		nil,
+		nil,
+		nil,
+	)
+
+	req := httptest.NewRequest("GET", "/content/verbose", nil)
+	res, err := app.Test(req, -1)
+	require.NoError(t, err)
+	body, err := io.ReadAll(res.Body)
+	require.NoError(t, err)
+
+	require.Equal(t, 200, res.StatusCode)
+	require.JSONEq(t, `{"data":null}`, string(body))
+}
+
+func contentAssetTestApp() *ApiServer {
+	app := &ApiServer{
+		App:        fiber.New(),
+		logger:     zap.NewNop(),
+		validators: NewNodes(),
+	}
+	app.Get("/content", app.contentNodes)
+	app.Get("/content/verbose", app.contentNodes)
+	app.Get("/content/:cid", app.contentAssetRedirect)
+	app.Get("/content/:cid/:asset", app.contentAssetRedirect)
+	return app
+}

api/server.go

@@ -791,6 +791,8 @@ func NewApiServer(config config.Config) *ApiServer {
 	app.Get("/content-nodes", app.contentNodes)
 	app.Get("/content/verbose", app.contentNodes)
 	app.Get("/content-nodes/verbose", app.contentNodes)
+	app.Get("/content/:cid", app.contentAssetRedirect)
+	app.Get("/content/:cid/:asset", app.contentAssetRedirect)
 
 	// Plans React app - serve static assets first, then SPA routing
 	app.Static("/plans/assets", "./static/plans/dist/assets")