From 2ededf47c8e318cbba299e306539b34ee476c092 Mon Sep 17 00:00:00 2001 From: HINS Date: Thu, 16 Jul 2026 23:33:22 +0800 Subject: [PATCH 1/3] feat(huggingface): add huggingface cloud storage driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 实现了HuggingFace仓库作为存储驱动的完整功能，支持模型/数据集/空间三种仓库类型，支持代理配置、LFS大文件上传、目录创建和删除等操作，同时在drivers/all.go中注册该驱动。 --- drivers/all.go | 4 +- drivers/huggingface/driver.go | 505 ++++++++++++++++++++++++++++++++++ drivers/huggingface/meta.go | 27 ++ drivers/huggingface/types.go | 99 +++++++ drivers/huggingface/util.go | 59 ++++ 5 files changed, 691 insertions(+), 3 deletions(-) create mode 100644 drivers/huggingface/driver.go create mode 100644 drivers/huggingface/meta.go create mode 100644 drivers/huggingface/types.go create mode 100644 drivers/huggingface/util.go diff --git a/drivers/all.go b/drivers/all.go index 4eb7987dca..34e4fb8772 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -13,7 +13,6 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/189_tv" _ "github.com/OpenListTeam/OpenList/v4/drivers/189pc" _ "github.com/OpenListTeam/OpenList/v4/drivers/alias" - _ "github.com/OpenListTeam/OpenList/v4/drivers/alidoc" _ "github.com/OpenListTeam/OpenList/v4/drivers/alist_v3" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_open" @@ -24,7 +23,6 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo" _ "github.com/OpenListTeam/OpenList/v4/drivers/chaoxing" _ "github.com/OpenListTeam/OpenList/v4/drivers/chunk" - _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudflare_imgbed" _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudreve" _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudreve_v4" _ "github.com/OpenListTeam/OpenList/v4/drivers/cnb_releases" @@ -34,7 +32,6 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_new" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/dropbox" - _ "github.com/OpenListTeam/OpenList/v4/drivers/emby" _ "github.com/OpenListTeam/OpenList/v4/drivers/febbox" _ "github.com/OpenListTeam/OpenList/v4/drivers/ftp" _ "github.com/OpenListTeam/OpenList/v4/drivers/github" @@ -43,6 +40,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/google_photo" _ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud" _ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud_open" + _ "github.com/OpenListTeam/OpenList/v4/drivers/huggingface" _ "github.com/OpenListTeam/OpenList/v4/drivers/ilanzou" _ "github.com/OpenListTeam/OpenList/v4/drivers/ipfs_api" _ "github.com/OpenListTeam/OpenList/v4/drivers/kodbox" diff --git a/drivers/huggingface/driver.go b/drivers/huggingface/driver.go new file mode 100644 index 0000000000..ef6bd641b9 --- /dev/null +++ b/drivers/huggingface/driver.go @@ -0,0 +1,505 @@ +package huggingface + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/base64" + "fmt" + "io" + "net" + "net/http" + "os" + stdpath "path" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/pkg/errors" +) + +// hfClient is a dedicated HTTP client for HF API requests, +// separate from base.HttpClient to avoid keep-alive connection reuse issues. +func hfClient() *http.Client { + return &http.Client{ + Timeout: 30 * time.Minute, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: false}, + DisableKeepAlives: true, + MaxIdleConns: 0, + }, + } +} + +const lfsThreshold = 5 << 20 + +type HuggingFace struct { + model.Storage + Addition + client *resty.Client +} + +func (d *HuggingFace) Config() driver.Config { + return config +} + +func (d *HuggingFace) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *HuggingFace) Init(ctx context.Context) error { + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + if d.RepoID == "" { + return errors.New("repo_id is required") + } + if d.RepoType == "" { + d.RepoType = "model" + } + if d.Ref == "" { + d.Ref = "main" + } + d.client = base.NewRestyClient(). + SetHeader("Accept", "application/json"). + SetDebug(false) + return nil +} + +func (d *HuggingFace) Drop(ctx context.Context) error { + return nil +} + +func (d *HuggingFace) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + path := dir.GetPath() + res, err := d.request().Get(d.apiURL(fmt.Sprintf("/tree/%s%s", d.Ref, path))) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toHFError(res) + } + var entries []TreeEntry + if err = utils.Json.Unmarshal(res.Body(), &entries); err != nil { + return nil, err + } + objs := make([]model.Obj, 0, len(entries)) + for _, entry := range entries { + objs = append(objs, entry.toModelObj()) + } + return objs, nil +} + +func (d *HuggingFace) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + url := d.resolveURL(d.Ref, file.GetPath()) + if proxy := strings.TrimSpace(d.Addition.HFProxy); proxy != "" { + url = strings.Replace(url, hfAPIBase, proxy, 1) + } + return &model.Link{ + URL: url, + }, nil +} + +func (d *HuggingFace) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + filePath := relativePath(stdpath.Join(dstDir.GetPath(), stream.GetName())) + fileName := stream.GetName() + + tmp, size, sha256Hex, err := d.saveStream(ctx, stream, up) + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + defer tmp.Close() + + sample, err := d.readSample(tmp) + if err != nil { + return err + } + + entries, err := d.preupload([]PreuploadFile{{ + Path: filePath, Size: size, Sample: sample, + }}) + if err != nil { + return err + } + + isLFS := size >= lfsThreshold + if len(entries) > 0 && entries[0].UploadMode == "lfs" { + isLFS = true + } + + if isLFS { + return d.lfsUploadAndCommit(ctx, tmp, filePath, fileName, sha256Hex, size, up) + } + return d.streamCommit(ctx, tmp, filePath, fileName, size, up) +} + +// saveStream writes stream to a temp file, returning the file, its size and sha256 hex. +func (d *HuggingFace) saveStream(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) (*os.File, int64, string, error) { + tmp, err := os.CreateTemp("", "hf-upload-*") + if err != nil { + return nil, 0, "", err + } + hasher := sha256.New() + writer := io.MultiWriter(tmp, hasher) + if err = utils.CopyWithCtx(ctx, writer, stream, 0, up); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return nil, 0, "", err + } + fi, err := tmp.Stat() + if err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return nil, 0, "", err + } + expectedSize := stream.GetSize() + if expectedSize > 0 && fi.Size() != expectedSize { + tmp.Close() + os.Remove(tmp.Name()) + return nil, 0, "", fmt.Errorf("saveStream: wrote %d bytes to temp file but stream.GetSize()=%d", fi.Size(), expectedSize) + } + size := fi.Size() + return tmp, size, fmt.Sprintf("%x", hasher.Sum(nil)), nil +} + +// streamCommit streams file content as base64 via NDJSON commit API. +func (d *HuggingFace) streamCommit(ctx context.Context, tmp *os.File, filePath, fileName string, size int64, up driver.UpdateProgress) error { + headerLine := fmt.Sprintf(`{"key":"header","value":{"summary":"Upload %s"}}`, fileName) + "\n" + filePrefix := fmt.Sprintf(`{"key":"file","value":{"path":"%s","content":"`, filePath) + fileSuffix := `","encoding":"base64"}}` + "\n" + + b64Len := calculateBase64Length(size) + contentLength := int64(len(headerLine)+len(filePrefix)) + b64Len + int64(len(fileSuffix)) + + pr, pw := io.Pipe() + go func() { + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + _ = pw.CloseWithError(err) + return + } + encoder := base64.NewEncoder(base64.StdEncoding, pw) + if _, err := io.Copy(encoder, tmp); err != nil { + _ = pw.CloseWithError(err) + return + } + _ = encoder.Close() + _ = pw.Close() + }() + + body := io.MultiReader( + strings.NewReader(headerLine), + strings.NewReader(filePrefix), + pr, + strings.NewReader(fileSuffix), + ) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + d.apiURL(fmt.Sprintf("/commit/%s", d.Ref)), + driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{Reader: body, Size: contentLength}, + UpdateProgress: up, + })) + if err != nil { + return fmt.Errorf("stream commit request build: %w", err) + } + req.Header.Set("Content-Type", "application/x-ndjson") + if d.ApiToken != "" { + req.Header.Set("Authorization", "Bearer "+d.ApiToken) + } + req.ContentLength = contentLength + + res, err := hfClient().Do(req) + if err != nil { + return fmt.Errorf("stream commit request: %w", err) + } + defer res.Body.Close() + if res.StatusCode != 200 { + bodyBytes, _ := io.ReadAll(res.Body) + return fmt.Errorf("commit failed: %s - %s", res.Status, string(bodyBytes)) + } + return nil +} + +// lfsUploadAndCommit handles LFS file upload and NDJSON commit with lfsFile reference. +func (d *HuggingFace) lfsUploadAndCommit(ctx context.Context, tmp *os.File, filePath, fileName, sha256Hex string, size int64, up driver.UpdateProgress) error { + batchURL := fmt.Sprintf("%s/%s.git/info/lfs/objects/batch", d.repoBase(), d.RepoID) + batchReq := LFSBatchRequest{ + Operation: "upload", Transfers: []string{"basic"}, + HashAlgo: "sha256", Ref: LFSRef{Name: d.Ref}, + Objects: []LFSObject{{OID: sha256Hex, Size: size}}, + } + lfsReq := d.client.R(). + SetHeader("Accept", "application/vnd.git-lfs+json"). + SetHeader("Content-Type", "application/vnd.git-lfs+json") + if d.ApiToken != "" { + lfsReq.SetHeader("Authorization", "Bearer "+d.ApiToken) + } + res, err := lfsReq.SetBody(batchReq).Post(batchURL) + if err != nil { + return err + } + if res.StatusCode() != 200 { + return toHFError(res) + } + var batchResp LFSBatchResponse + if err = utils.Json.Unmarshal(res.Body(), &batchResp); err != nil { + return err + } + if len(batchResp.Objects) == 0 { + return errors.New("lfs batch returned empty objects") + } + obj := batchResp.Objects[0] + if obj.Actions == nil { + return d.doCommitLFS(ctx, filePath, sha256Hex, size) + } + uploadAction, ok := obj.Actions["upload"] + if !ok { + return d.doCommitLFS(ctx, filePath, sha256Hex, size) + } + + // Verify temp file has the expected content before uploading to S3. + fi, statErr := tmp.Stat() + if statErr != nil { + return fmt.Errorf("failed to stat temp file before s3 upload: %w", statErr) + } + if fi.Size() != size { + return fmt.Errorf("temp file size %d does not match expected size %d: stream may have returned 0 bytes", fi.Size(), size) + } + if err = d.streamUpload(ctx, tmp, uploadAction, size, up); err != nil { + return err + } + + if verifyAction, ok := obj.Actions["verify"]; ok { + verifyBody := map[string]interface{}{"oid": sha256Hex, "size": size} + verifyReq := d.client.R().SetBody(verifyBody) + for k, v := range verifyAction.Header { + verifyReq.SetHeader(k, fmt.Sprint(v)) + } + verifyRes, err := verifyReq.Post(verifyAction.Href) + if err != nil { + return err + } + if verifyRes.StatusCode() > 299 { + return fmt.Errorf("lfs verify failed: %s", verifyRes.Status()) + } + } + + return d.doCommitLFS(ctx, filePath, sha256Hex, size) +} + +var s3HTTPClient *http.Client +var s3HTTPClientOnce sync.Once + +// s3Client returns a shared HTTP client for S3 LFS uploads. +func s3Client() *http.Client { + s3HTTPClientOnce.Do(func() { + s3HTTPClient = &http.Client{ + Timeout: 30 * time.Minute, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: false, + MinVersion: tls.VersionTLS12, + }, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: false, + MaxIdleConns: 2, + MaxIdleConnsPerHost: 2, + IdleConnTimeout: 90 * time.Second, + ResponseHeaderTimeout: 5 * time.Minute, + ExpectContinueTimeout: 5 * time.Second, + }, + } + }) + return s3HTTPClient +} + +// streamUpload streams file content to the LFS upload URL. +func (d *HuggingFace) streamUpload(ctx context.Context, tmp *os.File, action LFSAction, size int64, up driver.UpdateProgress) error { + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return err + } + // Use WithoutCancel for the body so the S3 upload completes even if ctx is canceled. + uploadCtx := context.WithoutCancel(ctx) + req, err := http.NewRequestWithContext(uploadCtx, http.MethodPut, action.Href, + driver.NewLimitedUploadStream(uploadCtx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{Reader: tmp, Size: size}, + UpdateProgress: up, + })) + if err != nil { + return fmt.Errorf("s3 upload request build: %w", err) + } + for k, v := range action.Header { + // Skip x-amz-* headers that were not signed (X-Amz-SignedHeaders=host) + if strings.HasPrefix(strings.ToLower(k), "x-amz-") { + continue + } + req.Header.Set(k, fmt.Sprint(v)) + } + if req.Header.Get("Content-Type") == "" { + req.Header.Set("Content-Type", "application/octet-stream") + } + req.ContentLength = size + + res, err := s3Client().Do(req) + if err != nil { + return fmt.Errorf("s3 upload request: %w", err) + } + defer res.Body.Close() + if res.StatusCode == 501 { + return d.streamUploadPost(ctx, tmp, action, size, up) + } + if res.StatusCode > 299 { + bodyBytes, _ := io.ReadAll(res.Body) + return fmt.Errorf("lfs upload failed: %s - %s", res.Status, string(bodyBytes)) + } + return nil +} + +// streamUploadPost retries upload with POST when PUT returns 501. +func (d *HuggingFace) streamUploadPost(ctx context.Context, tmp *os.File, action LFSAction, size int64, up driver.UpdateProgress) error { + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return err + } + uploadCtx := context.WithoutCancel(ctx) + req, err := http.NewRequestWithContext(uploadCtx, http.MethodPost, action.Href, + driver.NewLimitedUploadStream(uploadCtx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{Reader: tmp, Size: size}, + UpdateProgress: up, + })) + if err != nil { + return fmt.Errorf("s3 upload post request build: %w", err) + } + for k, v := range action.Header { + if strings.HasPrefix(strings.ToLower(k), "x-amz-") { + continue + } + req.Header.Set(k, fmt.Sprint(v)) + } + if req.Header.Get("Content-Type") == "" { + req.Header.Set("Content-Type", "application/octet-stream") + } + req.ContentLength = size + + res, err := s3Client().Do(req) + if err != nil { + return fmt.Errorf("s3 upload post request: %w", err) + } + defer res.Body.Close() + if res.StatusCode > 299 { + bodyBytes, _ := io.ReadAll(res.Body) + return fmt.Errorf("lfs upload failed: %s - %s", res.Status, string(bodyBytes)) + } + return nil +} + +// doCommitLFS sends a NDJSON commit with a lfsFile reference. +func (d *HuggingFace) doCommitLFS(ctx context.Context, filePath, sha256Hex string, size int64) error { + body := fmt.Sprintf(`{"key":"header","value":{"summary":"Upload %s"}}`+"\n"+`{"key":"lfsFile","value":{"path":"%s","algo":"sha256","oid":"%s","size":%d}}`+"\n", + stdpath.Base(filePath), filePath, sha256Hex, size) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + d.apiURL(fmt.Sprintf("/commit/%s", d.Ref)), + strings.NewReader(body)) + if err != nil { + return fmt.Errorf("commit request build: %w", err) + } + req.Header.Set("Content-Type", "application/x-ndjson") + if d.ApiToken != "" { + req.Header.Set("Authorization", "Bearer "+d.ApiToken) + } + req.ContentLength = int64(len(body)) + + res, err := hfClient().Do(req) + if err != nil { + return fmt.Errorf("commit request: %w", err) + } + defer res.Body.Close() + if res.StatusCode != 200 { + bodyBytes, _ := io.ReadAll(res.Body) + return fmt.Errorf("commit failed: %s - %s", res.Status, string(bodyBytes)) + } + return nil +} + +func (d *HuggingFace) preupload(files []PreuploadFile) ([]PreuploadResponseEntry, error) { + body := map[string]interface{}{"files": files} + res, err := d.request().SetBody(body).Post(d.apiURL(fmt.Sprintf("/preupload/%s", d.Ref))) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toHFError(res) + } + var resp PreuploadResponse + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + return resp.Files, nil +} + +func (d *HuggingFace) repoBase() string { + if d.RepoType == "model" { + return hfAPIBase + } + return hfAPIBase + "/" + apiRepoType(d.RepoType) +} + +func (d *HuggingFace) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + path := relativePath(stdpath.Join(parentDir.GetPath(), dirName) + "/.gitkeep") + body := map[string]interface{}{ + "summary": fmt.Sprintf("Create directory %s", dirName), + "files": []map[string]interface{}{ + {"path": path, "content": ""}, + }, + } + res, err := d.request().SetBody(body).Post(d.apiURL(fmt.Sprintf("/commit/%s", d.Ref))) + if err != nil { + return err + } + if res.StatusCode() != 200 { + return toHFError(res) + } + return nil +} + +func (d *HuggingFace) Remove(ctx context.Context, obj model.Obj) error { + body := map[string]interface{}{ + "summary": fmt.Sprintf("Delete %s", obj.GetName()), + "deletedEntries": []map[string]interface{}{ + {"path": relativePath(obj.GetPath())}, + }, + } + res, err := d.request().SetBody(body).Post(d.apiURL(fmt.Sprintf("/commit/%s", d.Ref))) + if err != nil { + return err + } + if res.StatusCode() != 200 { + return toHFError(res) + } + return nil +} + +func (d *HuggingFace) readSample(tmp *os.File) (string, error) { + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return "", err + } + buf := make([]byte, 512) + n, err := tmp.Read(buf) + if err != nil && err != io.EOF { + return "", err + } + return base64.StdEncoding.EncodeToString(buf[:n]), nil +} + +func calculateBase64Length(inputLength int64) int64 { + return 4 * ((inputLength + 2) / 3) +} + +var _ driver.Driver = (*HuggingFace)(nil) diff --git a/drivers/huggingface/meta.go b/drivers/huggingface/meta.go new file mode 100644 index 0000000000..d8645c0227 --- /dev/null +++ b/drivers/huggingface/meta.go @@ -0,0 +1,27 @@ +package huggingface + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootPath + ApiToken string `json:"api_token" type:"string" help:"HuggingFace API token, optional for public repos"` + RepoID string `json:"repo_id" type:"string" required:"true" help:"Repository ID, e.g. username/repo_name"` + Ref string `json:"ref" type:"string" help:"Branch, tag or commit SHA, main branch by default"` + RepoType string `json:"repo_type" type:"select" options:"model,dataset,space" default:"model" help:"Repository type"` + HFProxy string `json:"hf_proxy" type:"string" help:"HuggingFace proxy, e.g. https://hf-mirror.com"` +} + +var config = driver.Config{ + Name: "HuggingFace", + LocalSort: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &HuggingFace{} + }) +} diff --git a/drivers/huggingface/types.go b/drivers/huggingface/types.go new file mode 100644 index 0000000000..7d9b665f48 --- /dev/null +++ b/drivers/huggingface/types.go @@ -0,0 +1,99 @@ +package huggingface + +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type TreeEntry struct { + Type string `json:"type"` + Path string `json:"path"` + Size int64 `json:"size"` + SHA string `json:"sha"` +} + +func (e *TreeEntry) toModelObj() *model.Object { + obj := &model.Object{ + Name: e.Name(), + Size: e.Size, + Modified: time.Unix(0, 0), + IsFolder: e.Type == "directory", + Path: utils.FixAndCleanPath(e.Path), + } + if obj.IsFolder { + obj.Size = 0 + } + return obj +} + +func (e *TreeEntry) Name() string { + if idx := lastIndexByte(e.Path, '/'); idx >= 0 { + return e.Path[idx+1:] + } + return e.Path +} + +type ErrorResponse struct { + Error string `json:"error"` +} + +type PreuploadFile struct { + Path string `json:"path"` + Size int64 `json:"size"` + Sample string `json:"sample"` +} + +type PreuploadResponseEntry struct { + Path string `json:"path"` + UploadMode string `json:"uploadMode"` + ShouldIgnore bool `json:"shouldIgnore"` + OID string `json:"oid"` +} + +type PreuploadResponse struct { + Files []PreuploadResponseEntry `json:"files"` +} + +type LFSRef struct { + Name string `json:"name"` +} + +type LFSObject struct { + OID string `json:"oid"` + Size int64 `json:"size"` +} + +type LFSBatchRequest struct { + Operation string `json:"operation"` + Transfers []string `json:"transfers"` + HashAlgo string `json:"hash_algo"` + Ref LFSRef `json:"ref"` + Objects []LFSObject `json:"objects"` +} + +type LFSAction struct { + Href string `json:"href"` + Header map[string]interface{} `json:"header"` +} + +type LFSBatchObject struct { + OID string `json:"oid"` + Size int64 `json:"size"` + Actions map[string]LFSAction `json:"actions"` +} + +type LFSBatchResponse struct { + Transfer string `json:"transfer"` + Objects []LFSBatchObject `json:"objects"` +} + +func lastIndexByte(s string, c byte) int { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == c { + return i + } + } + return -1 +} diff --git a/drivers/huggingface/util.go b/drivers/huggingface/util.go new file mode 100644 index 0000000000..c237b542db --- /dev/null +++ b/drivers/huggingface/util.go @@ -0,0 +1,59 @@ +package huggingface + +import ( + "fmt" + "strings" + + "github.com/go-resty/resty/v2" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const hfAPIBase = "https://huggingface.co" + +func apiRepoType(t string) string { + switch t { + case "model": + return "models" + case "dataset": + return "datasets" + case "space": + return "spaces" + default: + return "models" + } +} + +func (d *HuggingFace) apiURL(path string) string { + return fmt.Sprintf("%s/api/%s/%s%s", hfAPIBase, apiRepoType(d.RepoType), d.RepoID, path) +} + +func (d *HuggingFace) resolveURL(ref, path string) string { + base := hfAPIBase + if d.RepoType != "model" { + base = hfAPIBase + "/" + apiRepoType(d.RepoType) + } + return fmt.Sprintf("%s/%s/resolve/%s/%s", base, d.RepoID, ref, strings.TrimPrefix(path, "/")) +} + +func (d *HuggingFace) request() *resty.Request { + req := d.client.R() + if d.ApiToken != "" { + req.SetHeader("Authorization", "Bearer "+d.ApiToken) + } + return req +} + +func relativePath(path string) string { + return strings.TrimPrefix(path, "/") +} + +func toHFError(res *resty.Response) error { + var errResp ErrorResponse + if err := utils.Json.Unmarshal(res.Body(), &errResp); err != nil { + return fmt.Errorf("%s", res.Status()) + } + if errResp.Error != "" { + return fmt.Errorf("%s: %s", res.Status(), errResp.Error) + } + return fmt.Errorf("%s", res.Status()) +} From 8f32a587bdd4994b4d92429732b711649f9ea13e Mon Sep 17 00:00:00 2001 From: HINS Date: Thu, 16 Jul 2026 23:43:20 +0800 Subject: [PATCH 2/3] feat(drivers): add alidoc, cloudflare_imgbed and emby drivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 重新导入三个编写过程中因意外丢失的存储驱动 --- .tmp/CloudPaste | 1 + .tmp/OpenList-Frontend | 1 + drivers/all.go | 3 +++ 3 files changed, 5 insertions(+) create mode 160000 .tmp/CloudPaste create mode 160000 .tmp/OpenList-Frontend diff --git a/.tmp/CloudPaste b/.tmp/CloudPaste new file mode 160000 index 0000000000..282ea1c7fb --- /dev/null +++ b/.tmp/CloudPaste @@ -0,0 +1 @@ +Subproject commit 282ea1c7fbbcd4fab40527664bf7479fd5bb922d diff --git a/.tmp/OpenList-Frontend b/.tmp/OpenList-Frontend new file mode 160000 index 0000000000..655a589481 --- /dev/null +++ b/.tmp/OpenList-Frontend @@ -0,0 +1 @@ +Subproject commit 655a5894819471a805d3d3646982299f096d6767 diff --git a/drivers/all.go b/drivers/all.go index 34e4fb8772..9c08ab7f53 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -13,6 +13,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/189_tv" _ "github.com/OpenListTeam/OpenList/v4/drivers/189pc" _ "github.com/OpenListTeam/OpenList/v4/drivers/alias" + _ "github.com/OpenListTeam/OpenList/v4/drivers/alidoc" _ "github.com/OpenListTeam/OpenList/v4/drivers/alist_v3" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_open" @@ -23,6 +24,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo" _ "github.com/OpenListTeam/OpenList/v4/drivers/chaoxing" _ "github.com/OpenListTeam/OpenList/v4/drivers/chunk" + _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudflare_imgbed" _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudreve" _ "github.com/OpenListTeam/OpenList/v4/drivers/cloudreve_v4" _ "github.com/OpenListTeam/OpenList/v4/drivers/cnb_releases" @@ -32,6 +34,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_new" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/dropbox" + _ "github.com/OpenListTeam/OpenList/v4/drivers/emby" _ "github.com/OpenListTeam/OpenList/v4/drivers/febbox" _ "github.com/OpenListTeam/OpenList/v4/drivers/ftp" _ "github.com/OpenListTeam/OpenList/v4/drivers/github" From 0bcebde47b1a84a1f9aa9d6ad7b28bcddedb656a Mon Sep 17 00:00:00 2001 From: HINS Date: Thu, 16 Jul 2026 23:59:05 +0800 Subject: [PATCH 3/3] =?UTF-8?q?chore:=20=E5=88=A0=E9=99=A4=E4=B8=A4?= =?UTF-8?q?=E4=B8=AA=E4=B8=B4=E6=97=B6=E5=AD=90=E6=A8=A1=E5=9D=97=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 清理仓库中不再需要的.tmp/CloudPaste和.tmp/OpenList-Frontend临时子模块 --- .tmp/CloudPaste | 1 - .tmp/OpenList-Frontend | 1 - 2 files changed, 2 deletions(-) delete mode 160000 .tmp/CloudPaste delete mode 160000 .tmp/OpenList-Frontend diff --git a/.tmp/CloudPaste b/.tmp/CloudPaste deleted file mode 160000 index 282ea1c7fb..0000000000 --- a/.tmp/CloudPaste +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 282ea1c7fbbcd4fab40527664bf7479fd5bb922d diff --git a/.tmp/OpenList-Frontend b/.tmp/OpenList-Frontend deleted file mode 160000 index 655a589481..0000000000 --- a/.tmp/OpenList-Frontend +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 655a5894819471a805d3d3646982299f096d6767