From e4f98f143ede9292ed15cbc2af15fa446ad8227a Mon Sep 17 00:00:00 2001 From: Raymond Jacobson Date: Tue, 30 Jun 2026 15:28:39 -0700 Subject: [PATCH] Add content asset redirect gateway --- api/content_asset.go | 109 ++++++++++++++++++++++++++++++++ api/content_asset_test.go | 129 ++++++++++++++++++++++++++++++++++++++ api/server.go | 2 + 3 files changed, 240 insertions(+) create mode 100644 api/content_asset.go create mode 100644 api/content_asset_test.go diff --git a/api/content_asset.go b/api/content_asset.go new file mode 100644 index 00000000..91a4ac24 --- /dev/null +++ b/api/content_asset.go @@ -0,0 +1,109 @@ +package api + +import ( + "net/url" + "slices" + "strings" + + "api.audius.co/config" + "api.audius.co/rendezvous" + "github.com/gofiber/fiber/v2" + "go.uber.org/zap" +) + +const contentAssetUpstreamHeader = "X-Audius-Upstream" + +func (app *ApiServer) contentAssetRedirect(c *fiber.Ctx) error { + cid := c.Params("cid") + asset := c.Params("asset") + + if !isValidContentPathSegment(cid) || (asset != "" && !isValidContentPathSegment(asset)) { + return fiber.NewError(fiber.StatusBadRequest, "invalid content path") + } + + target, upstream, ok := contentAssetTarget(cid, asset, string(c.Context().URI().QueryString())) + if !ok { + return fiber.NewError(fiber.StatusServiceUnavailable, "no content nodes available") + } + + c.Locals("upstream", upstream) + c.Set(contentAssetUpstreamHeader, upstream) + c.Set(fiber.HeaderCacheControl, "public, max-age=300") + app.logger.Debug("content asset redirect", + zap.String("cid", cid), + zap.String("asset", asset), + zap.String("upstream", upstream), + ) + + return c.Redirect(target, fiber.StatusTemporaryRedirect) +} + +func contentAssetTarget(cid string, asset string, rawQuery string) (string, string, bool) { + for _, endpoint := range contentAssetCandidates(cid) { + target, ok := contentAssetURL(endpoint, cid, asset, rawQuery) + if ok { + return target, endpoint, true + } + } + return "", "", false +} + +func contentAssetCandidates(cid string) []string { + candidates := make([]string, 0, len(config.Cfg.StoreAllNodes)+4) + seen := map[string]bool{} + + appendCandidate := func(endpoint string) { + endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/") + if endpoint == "" || seen[endpoint] || isBlockedContentEndpoint(endpoint) { + return + } + u, err := url.Parse(endpoint) + if err != nil || u.Scheme == "" || u.Host == "" { + return + } + seen[endpoint] = true + candidates = append(candidates, endpoint) + } + + for _, endpoint := range config.Cfg.StoreAllNodes { + appendCandidate(endpoint) + } + + first, rest := rendezvous.GlobalHasher.Select(cid) + appendCandidate(first) + for _, endpoint := range rest { + appendCandidate(endpoint) + } + + return candidates +} + +func contentAssetURL(endpoint string, cid string, asset string, rawQuery string) (string, bool) { + u, err := url.Parse(endpoint) + if err != nil || u.Scheme == "" || u.Host == "" { + return "", false + } + + path := "/content/" + url.PathEscape(cid) + if asset != "" { + path += "/" + url.PathEscape(asset) + } + u.Path = path + u.RawPath = "" + u.RawQuery = rawQuery + u.Fragment = "" + + return u.String(), true +} + +func isBlockedContentEndpoint(endpoint string) bool { + return slices.Contains(config.Cfg.BlacklistedNodes, endpoint) || + slices.Contains(config.Cfg.DeadNodes, endpoint) +} + +func isValidContentPathSegment(segment string) bool { + if segment == "" || segment == "." || segment == ".." { + return false + } + return !strings.ContainsAny(segment, `/\`+"\x00") +} diff --git a/api/content_asset_test.go b/api/content_asset_test.go new file mode 100644 index 00000000..bb70de18 --- /dev/null +++ b/api/content_asset_test.go @@ -0,0 +1,129 @@ +package api + +import ( + "io" + "net/http/httptest" + "testing" + + "api.audius.co/config" + "api.audius.co/rendezvous" + "github.com/gofiber/fiber/v2" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func withContentAssetConfig(t *testing.T, storeAll []string, blacklisted []string, dead []string, nodes []config.Node) { + t.Helper() + + oldStoreAll := config.Cfg.StoreAllNodes + oldBlacklisted := config.Cfg.BlacklistedNodes + oldDead := config.Cfg.DeadNodes + oldHasher := rendezvous.GlobalHasher + + config.Cfg.StoreAllNodes = storeAll + config.Cfg.BlacklistedNodes = blacklisted + config.Cfg.DeadNodes = dead + rendezvous.Refresh(nodes) + + t.Cleanup(func() { + config.Cfg.StoreAllNodes = oldStoreAll + config.Cfg.BlacklistedNodes = oldBlacklisted + config.Cfg.DeadNodes = oldDead + rendezvous.GlobalHasher = oldHasher + }) +} + +func TestContentAssetRedirectUsesStoreAllNode(t *testing.T) { + app := contentAssetTestApp() + withContentAssetConfig(t, + []string{"https://store-all.test", "https://blocked.test"}, + []string{"https://blocked.test"}, + nil, + []config.Node{{Endpoint: "https://validator.test"}}, + ) + + req := httptest.NewRequest("GET", "/content/test-cid/150x150.jpg?cache=bust", nil) + res, err := app.Test(req, -1) + require.NoError(t, err) + + require.Equal(t, 307, res.StatusCode) + require.Equal(t, "https://store-all.test/content/test-cid/150x150.jpg?cache=bust", res.Header.Get("Location")) + require.Equal(t, "https://store-all.test", res.Header.Get(contentAssetUpstreamHeader)) +} + +func TestContentAssetRedirectSupportsLegacyContentPath(t *testing.T) { + app := contentAssetTestApp() + withContentAssetConfig(t, + []string{"https://store-all.test"}, + nil, + nil, + nil, + ) + + req := httptest.NewRequest("GET", "/content/test-cid", nil) + res, err := app.Test(req, -1) + require.NoError(t, err) + + require.Equal(t, 307, res.StatusCode) + require.Equal(t, "https://store-all.test/content/test-cid", res.Header.Get("Location")) +} + +func TestContentAssetRedirectFallsBackToRendezvous(t *testing.T) { + app := contentAssetTestApp() + withContentAssetConfig(t, + nil, + nil, + nil, + []config.Node{{Endpoint: "https://validator.test"}}, + ) + + req := httptest.NewRequest("GET", "/content/test-cid/150x150.jpg", nil) + res, err := app.Test(req, -1) + require.NoError(t, err) + + require.Equal(t, 307, res.StatusCode) + require.Equal(t, "https://validator.test/content/test-cid/150x150.jpg", res.Header.Get("Location")) +} + +func TestContentAssetRedirectNoCandidates(t *testing.T) { + app := contentAssetTestApp() + withContentAssetConfig(t, nil, nil, nil, nil) + + req := httptest.NewRequest("GET", "/content/test-cid/150x150.jpg", nil) + res, err := app.Test(req, -1) + require.NoError(t, err) + + require.Equal(t, 503, res.StatusCode) +} + +func TestContentVerboseKeepsNodeListRoute(t *testing.T) { + app := contentAssetTestApp() + withContentAssetConfig(t, + []string{"https://store-all.test"}, + nil, + nil, + nil, + ) + + req := httptest.NewRequest("GET", "/content/verbose", nil) + res, err := app.Test(req, -1) + require.NoError(t, err) + body, err := io.ReadAll(res.Body) + require.NoError(t, err) + + require.Equal(t, 200, res.StatusCode) + require.JSONEq(t, `{"data":null}`, string(body)) +} + +func contentAssetTestApp() *ApiServer { + app := &ApiServer{ + App: fiber.New(), + logger: zap.NewNop(), + validators: NewNodes(), + } + app.Get("/content", app.contentNodes) + app.Get("/content/verbose", app.contentNodes) + app.Get("/content/:cid", app.contentAssetRedirect) + app.Get("/content/:cid/:asset", app.contentAssetRedirect) + return app +} diff --git a/api/server.go b/api/server.go index 021dd3b0..4b45716c 100644 --- a/api/server.go +++ b/api/server.go @@ -791,6 +791,8 @@ func NewApiServer(config config.Config) *ApiServer { app.Get("/content-nodes", app.contentNodes) app.Get("/content/verbose", app.contentNodes) app.Get("/content-nodes/verbose", app.contentNodes) + app.Get("/content/:cid", app.contentAssetRedirect) + app.Get("/content/:cid/:asset", app.contentAssetRedirect) // Plans React app - serve static assets first, then SPA routing app.Static("/plans/assets", "./static/plans/dist/assets")