diff --git a/drivers/all.go b/drivers/all.go index 601c7bfc5..46636f07c 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -55,6 +55,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/mediatrack" _ "github.com/OpenListTeam/OpenList/v4/drivers/mega" _ "github.com/OpenListTeam/OpenList/v4/drivers/misskey" + _ "github.com/OpenListTeam/OpenList/v4/drivers/modelscope" _ "github.com/OpenListTeam/OpenList/v4/drivers/mopan" _ "github.com/OpenListTeam/OpenList/v4/drivers/netease_music" _ "github.com/OpenListTeam/OpenList/v4/drivers/onedrive" diff --git a/drivers/modelscope/driver.go b/drivers/modelscope/driver.go new file mode 100644 index 000000000..1976af70a --- /dev/null +++ b/drivers/modelscope/driver.go @@ -0,0 +1,230 @@ +package modelscope + +import ( + "context" + "errors" + "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type ModelScope struct { + model.Storage + Addition +} + +func (d *ModelScope) Config() driver.Config { + return config +} + +func (d *ModelScope) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *ModelScope) Init(ctx context.Context) error { + if strings.TrimSpace(d.RepoID) == "" { + return errors.New("repo_id is required") + } + if strings.TrimSpace(d.Endpoint) == "" { + d.Endpoint = "https://modelscope.cn" + } + if strings.TrimSpace(d.Revision) == "" { + d.Revision = "master" + } + if d.RepoType == "" { + d.RepoType = "dataset" + } + if strings.TrimSpace(d.CommitMessage) == "" { + d.CommitMessage = "Upload from OpenList" + } + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + return nil +} + +func (d *ModelScope) Drop(ctx context.Context) error { + return nil +} + +// repoPath maps a storage obj path into a repo-relative path. +// +// The OpenList fs layer already prefixes RootFolderPath onto obj paths (the +// driver's GetRoot returns Path = RootFolderPath), so obj.GetPath() is already +// the repo-relative path. We only normalise it. +func (d *ModelScope) repoPath(objPath string) string { + return normalizePath(objPath) +} + +func (d *ModelScope) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + repoDir := d.repoPath(dir.GetPath()) + + // A directory listing only needs the direct children of repoDir. ModelScope + // returns the full recursive tree, so we fetch it (scoped to repoDir where + // the server supports it) and collapse to direct children. + entries, err := d.getFileList(ctx, repoDir, true) + if err != nil { + return nil, err + } + + prefix := repoDir + if prefix != "" { + prefix = utils.PathAddSeparatorSuffix(prefix) + } + + seenDirs := map[string]bool{} + objs := make([]model.Obj, 0, len(entries)) + for _, e := range entries { + ep := normalizePath(e.getPath()) + // Keep only entries directly inside repoDir. + if prefix != "" { + if !strings.HasPrefix(ep, prefix) { + continue + } + } + rel := strings.TrimPrefix(ep, prefix) + if rel == "" { + continue + } + // Skip deeper nesting: only take the first path segment. + seg := rel + if i := strings.Index(rel, "/"); i >= 0 { + seg = rel[:i] + } + + name := seg + isDir := e.isDir() || strings.Contains(rel, "/") + if isDir { + if seenDirs[name] { + continue + } + seenDirs[name] = true + } + + full := joinRepoPath(repoDir, name) + // Hide our placeholder for empty directories. + if name == ".gitkeep" { + continue + } + objs = append(objs, &model.Object{ + Name: name, + Size: e.getSize(), + Modified: toTime(e.modified()), + IsFolder: isDir, + Path: full, + ID: full, + }) + } + return objs, nil +} + +func (d *ModelScope) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + repoFile := d.repoPath(file.GetPath()) + return &model.Link{ + URL: d.downloadURL(repoFile), + Header: d.downloadHeaders(), + }, nil +} + +// getRootPath returns the storage root object for this driver. +func (d *ModelScope) GetRoot(ctx context.Context) (model.Obj, error) { + return &model.Object{ + Name: "root", + Path: d.RootFolderPath, + ID: d.RootFolderPath, + IsFolder: true, + }, nil +} + +func (d *ModelScope) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + // ModelScope (git-style) repositories cannot store empty directories, so we + // create a .gitkeep placeholder file. + dirPath := joinRepoPath(d.repoPath(parentDir.GetPath()), dirName) + keepPath := joinRepoPath(dirPath, ".gitkeep") + if err := d.putSmallFile(ctx, keepPath, nil); err != nil { + return nil, err + } + return &model.Object{ + Name: dirName, + Path: dirPath, + ID: dirPath, + IsFolder: true, + }, nil +} + +func (d *ModelScope) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + filePath := joinRepoPath(d.repoPath(dstDir.GetPath()), stream.GetName()) + obj, err := d.uploadFile(ctx, filePath, stream, up) + if err != nil { + return nil, err + } + return obj, nil +} + +func (d *ModelScope) Remove(ctx context.Context, obj model.Obj) error { + repoPath := d.repoPath(obj.GetPath()) + if obj.IsDir() { + // Removing a directory means removing every file under it. + entries, err := d.getFileList(ctx, repoPath, true) + if err != nil { + return err + } + prefix := repoPath + if prefix != "" { + prefix = utils.PathAddSeparatorSuffix(prefix) + } + var paths []string + for _, e := range entries { + ep := normalizePath(e.getPath()) + if ep == repoPath { + continue + } + if repoPath == "" || strings.HasPrefix(ep, prefix) { + if !e.isDir() { + paths = append(paths, ep) + } + } + } + if len(paths) == 0 { + return nil + } + return d.deleteFiles(ctx, paths) + } + return d.deleteFiles(ctx, []string{repoPath}) +} + +func (d *ModelScope) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + src := d.repoPath(srcObj.GetPath()) + dst := joinRepoPath(path.Dir(src), newName) + return d.movePath(ctx, src, dst, srcObj.IsDir()) +} + +func (d *ModelScope) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + src := d.repoPath(srcObj.GetPath()) + dst := joinRepoPath(d.repoPath(dstDir.GetPath()), srcObj.GetName()) + if dst == src { + return srcObj, nil + } + if strings.HasPrefix(dst, utils.PathAddSeparatorSuffix(src)) { + return nil, errors.New("cannot move a directory into itself") + } + return d.movePath(ctx, src, dst, srcObj.IsDir()) +} + +func (d *ModelScope) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + src := d.repoPath(srcObj.GetPath()) + dst := joinRepoPath(d.repoPath(dstDir.GetPath()), srcObj.GetName()) + if srcObj.IsDir() { + return nil, errs.NotSupport + } + // Copy a single file by downloading and re-uploading. + link, err := d.Link(ctx, srcObj, model.LinkArgs{}) + if err != nil { + return nil, err + } + return d.copyFileFromURL(ctx, src, dst, link) +} + +var _ driver.Driver = (*ModelScope)(nil) diff --git a/drivers/modelscope/meta.go b/drivers/modelscope/meta.go new file mode 100644 index 000000000..2ab0aaf3e --- /dev/null +++ b/drivers/modelscope/meta.go @@ -0,0 +1,40 @@ +package modelscope + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +// Addition holds the user-configurable fields of a ModelScope storage. +// +// A single ModelScope repository (a "repo file driver") maps to one OpenList +// storage mount. ModelScope stores files inside a git-style repository, so the +// natural mapping is: +// +// ModelScope RepoID + RepoType + Revision -> one OpenList storage +// +// See drivers/github for a repository-oriented driver with the same shape. +type Addition struct { + driver.RootPath + + Endpoint string `json:"endpoint" type:"select" options:"https://modelscope.cn,https://modelscope.ai" default:"https://modelscope.cn"` + Token string `json:"token" type:"string" required:"true" help:"ModelScope access token from https://modelscope.cn/my/access/token"` + RepoID string `json:"repo_id" type:"string" required:"true" help:"owner/repository, e.g. van/my-files"` + RepoType string `json:"repo_type" type:"select" options:"model,dataset" default:"dataset"` + Revision string `json:"revision" type:"string" default:"master" help:"Branch, tag or commit SHA"` + CommitMessage string `json:"commit_message" type:"string" default:"Upload from OpenList"` +} + +var config = driver.Config{ + Name: "ModelScope", + LocalSort: true, + DefaultRoot: "/", + CheckStatus: false, + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ModelScope{} + }) +} diff --git a/drivers/modelscope/types.go b/drivers/modelscope/types.go new file mode 100644 index 000000000..a77978cac --- /dev/null +++ b/drivers/modelscope/types.go @@ -0,0 +1,210 @@ +package modelscope + +import ( + stdpath "path" + "strings" + "time" +) + +// FileEntry is a single entry of the ModelScope file-tree listing. +// +// ModelScope's legacy API returns entries with PascalCase keys (Path, Size, +// Sha256, Type, ...) but the OpenAPI uses snake_case. We accept both so the +// driver keeps working whichever endpoint shape the server returns. +type FileEntry struct { + Path string `json:"Path"` + Name string `json:"Name"` + Type string `json:"Type"` + Size int64 `json:"Size"` + Sha256 string `json:"Sha256"` + LastModified any `json:"LastModified"` + CommittedDate any `json:"CommittedDate"` + IsLFS bool `json:"IsLFS"` + + // snake_case fallbacks (OpenAPI-style) + PathSnake string `json:"path"` + NameSnake string `json:"name"` + TypeSnake string `json:"type"` + SizeSnake int64 `json:"size"` + Sha256Snake string `json:"sha256"` + LastModifiedSnake any `json:"last_modified"` +} + +func (f *FileEntry) getPath() string { + if f.Path != "" { + return f.Path + } + return f.PathSnake +} + +func (f *FileEntry) getType() string { + if f.Type != "" { + return f.Type + } + return f.TypeSnake +} + +func (f *FileEntry) getSize() int64 { + if f.Size != 0 { + return f.Size + } + return f.SizeSnake +} + +func (f *FileEntry) isDir() bool { + t := f.getType() + return t == "tree" || t == "dir" +} + +// modified returns the best-available modification timestamp. +func (f *FileEntry) modified() any { + if f.LastModified != nil { + return f.LastModified + } + if f.CommittedDate != nil { + return f.CommittedDate + } + return f.LastModifiedSnake +} + +// fileListResp is the unwrapped body of a file-tree listing. ModelScope wraps +// the payload in a {"Data": {...}} envelope; the outer pagination fields +// (TotalCount/PageNumber/PageSize) live outside Data on the top-level object. +type fileListResp struct { + Files []FileEntry `json:"Files"` + TotalCount int `json:"TotalCount"` + PageNumber int `json:"PageNumber"` + PageSize int `json:"PageSize"` +} + +// fileListEnvelope is the full top-level response including the Data wrapper. +type fileListEnvelope struct { + Data fileListResp `json:"Data"` + TotalCount int `json:"TotalCount"` + PageNumber int `json:"PageNumber"` + PageSize int `json:"PageSize"` +} + +// rawFileList is used when the listing body is a bare JSON array. +type rawFileList []FileEntry + +type errResp struct { + Code int `json:"Code"` + Message string `json:"Message"` + Msg string `json:"message"` +} + +func (e *errResp) text() string { + if e.Message != "" { + return e.Message + } + return e.Msg +} + +// lfsBatchReq is the LFS batch API request body. +type lfsBatchReq struct { + Operation string `json:"operation"` + Objects []lfsObject `json:"objects"` +} + +type lfsObject struct { + Oid string `json:"oid"` + Size int64 `json:"size"` +} + +type lfsAction struct { + Href string `json:"href"` +} + +type lfsRespObject struct { + Oid string `json:"oid"` + Actions map[string]lfsAction `json:"actions"` +} + +type lfsBatchResp struct { + Objects []lfsRespObject `json:"objects"` + Data lfsBatchData `json:"Data"` +} + +// objects returns the blob objects, preferring the top-level "objects" field +// and falling back to the "Data.objects" envelope. +func (r *lfsBatchResp) objects() []lfsRespObject { + if len(r.Objects) > 0 { + return r.Objects + } + return r.Data.Objects +} + +type lfsBatchData struct { + Objects []lfsRespObject `json:"objects"` +} + +// commitAction is one file operation inside a commit request. +// +// action: create | update | delete +// type: normal | lfs +type commitAction struct { + Action string `json:"action"` + Path string `json:"path"` + Type string `json:"type"` + Size int64 `json:"size"` + Sha256 string `json:"sha256"` + Content string `json:"content"` + Encoding string `json:"encoding"` +} + +type commitReq struct { + CommitMessage string `json:"commit_message"` + Actions []commitAction `json:"actions"` +} + +// normalizePath returns a clean, repo-relative path (no leading slash) with +// forward slashes. An empty input or "/" maps to "" (the repository root). +// ModelScope repo paths are git-style relative paths, so the leading slash +// must be stripped. +func normalizePath(p string) string { + p = strings.ReplaceAll(p, "\\", "/") + p = stdpath.Clean(p) + // Strip a single leading slash to get a repo-relative path. + p = strings.TrimPrefix(p, "/") + if p == "." { + return "" + } + return p +} + +// joinRepoPath joins a parent directory with a child name into a repo-relative +// path (no leading slash). +func joinRepoPath(parent, name string) string { + if parent == "" { + return normalizePath(name) + } + return normalizePath(parent + "/" + name) +} + +// toTime converts the flexible LastModified field into a time.Time. +func toTime(v any) time.Time { + switch t := v.(type) { + case time.Time: + return t + case int64: + return time.Unix(t, 0) + case float64: + // ModelScope may return seconds or milliseconds. + if t > 1e12 { + return time.UnixMilli(int64(t)) + } + return time.Unix(int64(t), 0) + case string: + if t == "" { + return time.Time{} + } + parsed, err := time.Parse(time.RFC3339, t) + if err != nil { + return time.Time{} + } + return parsed + default: + return time.Time{} + } +} diff --git a/drivers/modelscope/upload.go b/drivers/modelscope/upload.go new file mode 100644 index 000000000..b58cf79ef --- /dev/null +++ b/drivers/modelscope/upload.go @@ -0,0 +1,485 @@ +package modelscope + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net/http" + "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/pkg/errors" +) + +const ( + // lfsThreshold mirrors modelscope_hub: files larger than 1 MiB are forced + // into LFS mode regardless of suffix. + lfsThreshold = 1 * 1024 * 1024 +) + +// uploadFile uploads a single file into the repository at filePath. +// +// It chooses "normal" (inline base64) or "lfs" (blob upload + pointer) mode +// based on size/suffix, matching modelscope_hub's _upload_mode. +func (d *ModelScope) uploadFile(ctx context.Context, filePath string, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // Materialise the stream into a seekable file so we can compute SHA256 and + // then re-read it for the actual upload without buffering twice. + file, err := stream.CacheFullAndWriter(&up, nil) + if err != nil { + return nil, err + } + + size := stream.GetSize() + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, err + } + digest, err := utils.HashFile(utils.SHA256, file) + if err != nil { + return nil, err + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, err + } + + mode := d.uploadMode(filePath, size) + switch mode { + case "lfs": + if err := d.uploadLFSFile(ctx, filePath, file, size, digest, up); err != nil { + return nil, err + } + default: + if err := d.uploadNormalFile(ctx, filePath, file, size, up); err != nil { + return nil, err + } + } + + return &model.Object{ + Name: path.Base(filePath), + Path: filePath, + ID: filePath, + Size: size, + HashInfo: utils.NewHashInfo(utils.SHA256, digest), + }, nil +} + +// uploadMode decides normal vs lfs, matching modelscope_hub._is_lfs. +func (d *ModelScope) uploadMode(filePath string, size int64) string { + if size > lfsThreshold { + return "lfs" + } + suffix := strings.ToLower(pathExt(filePath)) + if d.isLfsSuffix(suffix) { + return "lfs" + } + return "normal" +} + +// uploadNormalFile commits a small file inline as base64 content. +func (d *ModelScope) uploadNormalFile(ctx context.Context, filePath string, file model.File, size int64, up driver.UpdateProgress) error { + if _, err := file.Seek(0, io.SeekStart); err != nil { + return err + } + data, err := io.ReadAll(file) + if err != nil { + return err + } + content := base64.StdEncoding.EncodeToString(data) + action := commitAction{ + Action: "create", + Path: filePath, + Type: "normal", + Size: size, + Content: content, + Encoding: "base64", + } + return d.commit(ctx, []commitAction{action}) +} + +// uploadLFSFile uploads a large file via the LFS batch API and commits an LFS +// pointer. +func (d *ModelScope) uploadLFSFile(ctx context.Context, filePath string, file model.File, size int64, digest string, up driver.UpdateProgress) error { + // Ask the server which blobs are missing. + req := lfsBatchReq{ + Operation: "upload", + Objects: []lfsObject{{Oid: digest, Size: size}}, + } + var batchResp lfsBatchResp + res, err := base.NewRestyClient().R(). + SetHeaders(d.authHeaders()). + SetContext(ctx). + SetBody(req). + Post(d.buildURL("repos/" + d.repoSegment() + "/" + d.RepoID + "/info/lfs/objects/batch")) + if err != nil { + return err + } + if err := checkErr(res); err != nil { + return err + } + if err := jsonUnmarshal(res.Body(), &batchResp); err != nil { + return err + } + + // Find the upload href for our oid, if the server says we must upload. + var uploadURL string + for _, o := range batchResp.objects() { + if o.Oid == digest { + if a, ok := o.Actions["upload"]; ok { + uploadURL = a.Href + } + } + } + + if uploadURL != "" { + if err := d.putBlob(ctx, uploadURL, file, size, up); err != nil { + return err + } + } + + action := commitAction{ + Action: "create", + Path: filePath, + Type: "lfs", + Size: size, + Sha256: digest, + } + return d.commit(ctx, []commitAction{action}) +} + +// putBlob PUTs the file content to a presigned LFS upload URL. +func (d *ModelScope) putBlob(ctx context.Context, uploadURL string, file model.File, size int64, up driver.UpdateProgress) error { + if _, err := file.Seek(0, io.SeekStart); err != nil { + return err + } + reader := &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ + Reader: file, + Size: size, + }, + UpdateProgress: up, + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, reader) + if err != nil { + return err + } + req.Header.Set("Content-Length", fmt.Sprintf("%d", size)) + if t := strings.TrimSpace(d.Token); t != "" { + req.Header.Set("Authorization", "Bearer "+t) + req.Header.Set("Cookie", "m_session_id="+t) + } + + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return errors.Errorf("lfs blob upload failed %s: %s", resp.Status, string(body)) + } + return nil +} + +// commit posts a set of file operations to the commit endpoint. +func (d *ModelScope) commit(ctx context.Context, actions []commitAction) error { + if len(actions) == 0 { + return nil + } + body := commitReq{ + CommitMessage: d.CommitMessage, + Actions: actions, + } + res, err := base.NewRestyClient().R(). + SetHeaders(d.authHeaders()). + SetContext(ctx). + SetBody(body). + Post(d.buildURL("repos/" + d.repoSegment() + "/" + d.RepoID + "/commit/" + d.Revision)) + if err != nil { + return err + } + return checkErr(res) +} + +// deleteFiles removes one or more files via a single commit with delete actions. +func (d *ModelScope) deleteFiles(ctx context.Context, paths []string) error { + actions := make([]commitAction, 0, len(paths)) + for _, p := range paths { + actions = append(actions, commitAction{ + Action: "delete", + Path: p, + }) + } + return d.commit(ctx, actions) +} + +// movePath moves a file or directory by committing a create/delete pair. +// +// LFS files can be re-pointed by sha256 without re-downloading. Normal files +// must carry their base64 content inline, so we download them and re-upload. +func (d *ModelScope) movePath(ctx context.Context, src, dst string, isDir bool) (model.Obj, error) { + if isDir { + entries, err := d.getFileList(ctx, src, true) + if err != nil { + return nil, err + } + prefix := utils.PathAddSeparatorSuffix(src) + var actions []commitAction + for _, e := range entries { + ep := normalizePath(e.getPath()) + if !strings.HasPrefix(ep, prefix) { + continue + } + if e.isDir() { + continue + } + rel := strings.TrimPrefix(ep, prefix) + newPath := joinRepoPath(dst, rel) + sha := e.Sha256 + if sha == "" { + sha = e.Sha256Snake + } + if sha != "" { + // LFS file: re-point the blob. + actions = append(actions, commitAction{Action: "create", Path: newPath, Type: "lfs", Size: e.getSize(), Sha256: sha}) + } else { + // Normal file: download content and inline it. + content, err := d.fetchFileContent(ctx, ep) + if err != nil { + return nil, err + } + actions = append(actions, commitAction{ + Action: "create", + Path: newPath, + Type: "normal", + Size: e.getSize(), + Content: base64.StdEncoding.EncodeToString(content), + Encoding: "base64", + }) + } + actions = append(actions, commitAction{Action: "delete", Path: ep}) + } + if len(actions) > 0 { + if err := d.commit(ctx, actions); err != nil { + return nil, err + } + } + return &model.Object{ + Name: path.Base(dst), + Path: dst, + ID: dst, + IsFolder: true, + }, nil + } + + // Single file move. + link, err := d.downloadLinkForPath(ctx, src) + if err != nil { + return nil, err + } + // Download and re-upload. + newObj, err := d.copyFileFromURL(ctx, src, dst, link) + if err != nil { + return nil, err + } + if err := d.deleteFiles(ctx, []string{src}); err != nil { + return nil, err + } + return newObj, nil +} + +// fetchFileContent downloads a file's raw bytes for inline (normal-file) moves. +func (d *ModelScope) fetchFileContent(ctx context.Context, repoPath string) ([]byte, error) { + link := &model.Link{URL: d.downloadURL(repoPath), Header: d.downloadHeaders()} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil) + if err != nil { + return nil, err + } + for k, vs := range link.Header { + for _, v := range vs { + req.Header.Add(k, v) + } + } + resp, err := base.HttpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, errors.Errorf("download failed %s: %s", resp.Status, string(body)) + } + return io.ReadAll(resp.Body) +} + +// downloadLinkForPath returns a Link for the given repo path. +func (d *ModelScope) downloadLinkForPath(ctx context.Context, repoPath string) (*model.Link, error) { + return &model.Link{ + URL: d.downloadURL(repoPath), + Header: d.downloadHeaders(), + }, nil +} + +// copyFileFromURL downloads the file at src (via link) and re-uploads it to dst. +func (d *ModelScope) copyFileFromURL(ctx context.Context, src, dst string, link *model.Link) (model.Obj, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, link.URL, nil) + if err != nil { + return nil, err + } + for k, vs := range link.Header { + for _, v := range vs { + req.Header.Add(k, v) + } + } + resp, err := base.HttpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, errors.Errorf("download failed %s: %s", resp.Status, string(body)) + } + + h := sha256.New() + data, err := io.ReadAll(io.TeeReader(resp.Body, h)) + if err != nil { + return nil, err + } + digest := hex.EncodeToString(h.Sum(nil)) + size := int64(len(data)) + + obj := &model.Object{ + Name: path.Base(dst), + Path: dst, + ID: dst, + Size: size, + HashInfo: utils.NewHashInfo(utils.SHA256, digest), + } + + mode := d.uploadMode(dst, size) + if mode == "lfs" { + // Upload via LFS. + req := lfsBatchReq{Operation: "upload", Objects: []lfsObject{{Oid: digest, Size: size}}} + var batchResp lfsBatchResp + res, err := base.NewRestyClient().R(). + SetHeaders(d.authHeaders()). + SetContext(ctx). + SetBody(req). + Post(d.buildURL("repos/" + d.repoSegment() + "/" + d.RepoID + "/info/lfs/objects/batch")) + if err != nil { + return nil, err + } + if err := checkErr(res); err != nil { + return nil, err + } + if err := jsonUnmarshal(res.Body(), &batchResp); err != nil { + return nil, err + } + var uploadURL string + for _, o := range batchResp.objects() { + if o.Oid == digest { + if a, ok := o.Actions["upload"]; ok { + uploadURL = a.Href + } + } + } + if uploadURL != "" { + if err := d.putBytes(ctx, uploadURL, data, size); err != nil { + return nil, err + } + } + return obj, d.commit(ctx, []commitAction{{Action: "create", Path: dst, Type: "lfs", Size: size, Sha256: digest}}) + } + + // Normal inline upload. + content := base64.StdEncoding.EncodeToString(data) + return obj, d.commit(ctx, []commitAction{{Action: "create", Path: dst, Type: "normal", Size: size, Content: content, Encoding: "base64"}}) +} + +// putBytes uploads in-memory bytes to a presigned LFS URL. +func (d *ModelScope) putBytes(ctx context.Context, uploadURL string, data []byte, size int64) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, strings.NewReader(string(data))) + if err != nil { + return err + } + req.Header.Set("Content-Length", fmt.Sprintf("%d", size)) + if t := strings.TrimSpace(d.Token); t != "" { + req.Header.Set("Authorization", "Bearer "+t) + req.Header.Set("Cookie", "m_session_id="+t) + } + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return errors.Errorf("lfs blob upload failed %s: %s", resp.Status, string(body)) + } + return nil +} + +// putSmallFile writes an empty/near-empty file (used for .gitkeep placeholders). +func (d *ModelScope) putSmallFile(ctx context.Context, filePath string, _ []byte) error { + return d.commit(ctx, []commitAction{{ + Action: "create", + Path: filePath, + Type: "normal", + Size: 0, + Content: "", + Encoding: "base64", + }}) +} + +// pathExt returns the lowercased file extension (without dot). +func pathExt(p string) string { + return strings.ToLower(utils.Ext(p)) +} + +// isLfsSuffix reports whether a file suffix forces LFS mode for this repo type. +func (d *ModelScope) isLfsSuffix(suffix string) bool { + // These lists mirror modelscope_hub's MODEL_LFS_SUFFIX / DATASET_LFS_SUFFIX. + common := map[string]bool{ + ".7z": true, ".bin": true, ".bz2": true, ".gz": true, ".h5": true, + ".msgpack": true, ".npy": true, ".npz": true, ".ot": true, ".parquet": true, + ".pb": true, ".pickle": true, ".pkl": true, ".rar": true, ".tar": true, + ".tgz": true, ".wasm": true, ".zip": true, ".zst": true, ".joblib": true, + ".ftz": true, + } + if common["."+suffix] { + return true + } + if d.RepoType == "model" { + modelOnly := map[string]bool{ + ".ckpt": true, ".mlmodel": true, ".model": true, ".onnx": true, + ".pt": true, ".pth": true, ".safetensors": true, ".tflite": true, + ".xz": true, ".arrow": true, + } + return modelOnly["."+suffix] + } + // dataset + datasetOnly := map[string]bool{ + ".aac": true, ".audio": true, ".bmp": true, ".flac": true, ".gif": true, + ".jack": true, ".jpeg": true, ".jpg": true, ".jsonl": true, ".lz4": true, + ".pcm": true, ".raw": true, ".sam": true, ".wav": true, ".webm": true, + ".webp": true, ".tiff": true, ".mp3": true, ".mp4": true, ".ogg": true, + ".arrow": true, ".png": true, + } + return datasetOnly["."+suffix] +} + +var ( + _ driver.PutResult = (*ModelScope)(nil) + _ driver.MkdirResult = (*ModelScope)(nil) + _ driver.Remove = (*ModelScope)(nil) + _ driver.MoveResult = (*ModelScope)(nil) + _ driver.RenameResult = (*ModelScope)(nil) + _ driver.CopyResult = (*ModelScope)(nil) + _ driver.GetRooter = (*ModelScope)(nil) +) diff --git a/drivers/modelscope/util.go b/drivers/modelscope/util.go new file mode 100644 index 000000000..d0e0c385f --- /dev/null +++ b/drivers/modelscope/util.go @@ -0,0 +1,188 @@ +package modelscope + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/pkg/errors" +) + +const ( + legacyAPIPrefix = "/api/v1" +) + +// repoSegment returns the plural URL segment for a repo type ("model" -> +// "models", "dataset" -> "datasets"). +func (d *ModelScope) repoSegment() string { + t := d.RepoType + if t == "" { + t = "dataset" + } + if strings.HasSuffix(t, "s") { + return t + } + return t + "s" +} + +// endpoint returns the configured endpoint with a trailing slash removed. +func (d *ModelScope) endpoint() string { + return strings.TrimRight(d.Endpoint, "/") +} + +// buildURL constructs a full legacy-API URL from a repo-relative path. +func (d *ModelScope) buildURL(path string) string { + return d.endpoint() + legacyAPIPrefix + "/" + strings.TrimLeft(path, "/") +} + +// apiURL constructs a URL under /api/v1/{type}s/{repoID}/... +func (d *ModelScope) apiURL(suffix string) string { + return d.buildURL(d.repoSegment() + "/" + d.RepoID + "/" + suffix) +} + +// authHeaders returns the headers used for authenticated requests. ModelScope +// accepts a Bearer token and, for older endpoints, an m_session_id cookie. We +// set both to maximise compatibility (mirrors modelscope_hub _headers). +func (d *ModelScope) authHeaders() map[string]string { + h := map[string]string{ + "Content-Type": "application/json", + } + if t := strings.TrimSpace(d.Token); t != "" { + h["Authorization"] = "Bearer " + t + h["Cookie"] = "m_session_id=" + t + } + return h +} + +// checkErr inspects a resty response and returns a descriptive error on non-2xx. +func checkErr(res *resty.Response) error { + if res.StatusCode() >= 200 && res.StatusCode() < 300 { + return nil + } + var e errResp + if err := jsonUnmarshal(res.Body(), &e); err == nil && e.text() != "" { + return errors.Errorf("modelscope %s: %s", res.Status(), e.text()) + } + return errors.Errorf("modelscope %s: %s", res.Status(), string(res.Body())) +} + +// jsonUnmarshal is a small indirection so utils.Json is the single source of +// JSON handling (consistent with the rest of the drivers). +func jsonUnmarshal(data []byte, v any) error { + return utils.Json.Unmarshal(data, v) +} + +// getFileList fetches the (possibly recursive) file tree of the repository at +// the configured revision, following dataset pagination when present. +func (d *ModelScope) getFileList(ctx context.Context, root string, recursive bool) ([]FileEntry, error) { + var all []FileEntry + + if d.RepoType == "dataset" { + // Datasets paginate via the repo/tree endpoint (TotalCount/PageNumber). + page := 1 + pageSize := 200 + for { + entries, total, err := d.getFileListPage(ctx, root, recursive, page, pageSize) + if err != nil { + return nil, err + } + all = append(all, entries...) + if total > 0 && len(all) >= total { + break + } + if len(entries) < pageSize { + break + } + page++ + } + return all, nil + } + + // Models/other: repo/files does not paginate; it silently truncates at + // REPO_FILES_TRUNCATION_LIMIT. We fetch the single page. + entries, _, err := d.getFileListPage(ctx, root, recursive, 0, 0) + return entries, err +} + +// getFileListPage performs one listing request and returns the entries plus the +// total count (0 if the endpoint does not report it). +func (d *ModelScope) getFileListPage(ctx context.Context, root string, recursive bool, page, pageSize int) ([]FileEntry, int, error) { + params := map[string]string{ + "Revision": d.Revision, + "Recursive": fmt.Sprintf("%t", recursive), + } + if root != "" && root != "/" { + params["Root"] = root + } + + var suffix string + if d.RepoType == "dataset" { + suffix = "repo/tree" + params["PageNumber"] = fmt.Sprintf("%d", page) + params["PageSize"] = fmt.Sprintf("%d", pageSize) + } else { + suffix = "repo/files" + } + + res, err := base.NewRestyClient().R(). + SetHeaders(d.authHeaders()). + SetQueryParams(params). + SetContext(ctx). + Get(d.apiURL(suffix)) + if err != nil { + return nil, 0, err + } + if err := checkErr(res); err != nil { + return nil, 0, err + } + + // Try the full envelope (with Data wrapper + outer pagination fields). + var env fileListEnvelope + if err := jsonUnmarshal(res.Body(), &env); err == nil && env.Data.Files != nil { + total := env.TotalCount + if total == 0 { + total = env.Data.TotalCount + } + return env.Data.Files, total, nil + } + + // A bare array is also possible. + var raw rawFileList + if err := jsonUnmarshal(res.Body(), &raw); err == nil { + return []FileEntry(raw), 0, nil + } + + // Unwrapped file list (no Data wrapper). + var resp fileListResp + if err := jsonUnmarshal(res.Body(), &resp); err != nil { + return nil, 0, err + } + return resp.Files, resp.TotalCount, nil +} + +// downloadURL builds the raw download URL for a file. +// +// Pattern: {endpoint}/api/v1/{type}s/{repo_id}/repo?Revision={rev}&FilePath={path} +func (d *ModelScope) downloadURL(filePath string) string { + q := url.Values{} + q.Set("Revision", d.Revision) + q.Set("FilePath", filePath) + return d.apiURL("repo") + "?" + q.Encode() +} + +// downloadHeaders returns the headers needed when fetching a file. For public +// repos no auth is needed, but the token is harmless and required for private +// repos. +func (d *ModelScope) downloadHeaders() http.Header { + h := http.Header{} + if t := strings.TrimSpace(d.Token); t != "" { + h.Set("Authorization", "Bearer "+t) + h.Set("Cookie", "m_session_id="+t) + } + return h +}