From 46fb06db2e0040492445e03adb184f712640e5b8 Mon Sep 17 00:00:00 2001 From: ssongliu Date: Fri, 17 Jul 2026 10:39:40 +0800 Subject: [PATCH] refactor(agent): replace onedrive sdk with graph api --- agent/go.mod | 4 +- agent/go.sum | 6 - agent/utils/cloud_storage/client/onedrive.go | 530 +++++++++---------- frontend/src/views/cronjob/cronjob/index.vue | 4 +- 4 files changed, 266 insertions(+), 278 deletions(-) diff --git a/agent/go.mod b/agent/go.mod index 639ef32244a5..b8767e10e1bf 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -20,7 +20,6 @@ require ( github.com/go-redis/redis v6.15.9+incompatible github.com/go-resty/resty/v2 v2.17.2 github.com/go-sql-driver/mysql v1.10.0 - github.com/goh-chunlin/go-onedrive v1.1.1 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/jackc/pgx/v5 v5.10.0 @@ -52,7 +51,6 @@ require ( go.mongodb.org/mongo-driver/v2 v2.7.0 golang.org/x/crypto v0.53.0 golang.org/x/net v0.56.0 - golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.21.0 golang.org/x/sys v0.46.0 golang.org/x/text v0.39.0 @@ -134,7 +132,6 @@ require ( github.com/google/go-querystring v1.2.0 // indirect github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect - github.com/h2non/filetype v1.1.3 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.198 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -216,6 +213,7 @@ require ( go4.org v0.0.0-20260112195520-a5071408f32f // indirect golang.org/x/arch v0.26.0 // indirect golang.org/x/mod v0.37.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/tools v0.47.0 // indirect google.golang.org/protobuf v1.36.11 // indirect modernc.org/fileutil v1.4.0 // indirect diff --git a/agent/go.sum b/agent/go.sum index 2e9f7043e932..c816092ccf58 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -350,8 +350,6 @@ github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/goh-chunlin/go-onedrive v1.1.1 h1:HGtHk5iG0MZ92zYUtaY04czfZPBIJUr12UuFc+PW8m4= -github.com/goh-chunlin/go-onedrive v1.1.1/go.mod h1:N8qIGHD7tryO734epiBKk5oXcpGwxKET/u3LuBHciTs= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= @@ -446,9 +444,6 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= -github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= -github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= -github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -1061,7 +1056,6 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= diff --git a/agent/utils/cloud_storage/client/onedrive.go b/agent/utils/cloud_storage/client/onedrive.go index c34b72836833..ce5ef30dd6a2 100644 --- a/agent/utils/cloud_storage/client/onedrive.go +++ b/agent/utils/cloud_storage/client/onedrive.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "context" - "crypto/tls" "encoding/json" "errors" "fmt" @@ -17,12 +16,15 @@ import ( "strings" "github.com/1Panel-dev/1Panel/agent/utils/files" - odsdk "github.com/goh-chunlin/go-onedrive/onedrive" - "golang.org/x/oauth2" ) +const oneDriveGlobalBaseURL = "https://graph.microsoft.com/v1.0/" +const oneDriveChinaBaseURL = "https://microsoftgraph.chinacloudapi.cn/v1.0/" + type oneDriveClient struct { - client odsdk.Client + client *http.Client + baseURL *url.URL + token string } func NewOneDriveClient(vars map[string]interface{}) (*oneDriveClient, error) { @@ -30,83 +32,56 @@ func NewOneDriveClient(vars map[string]interface{}) (*oneDriveClient, error) { if err != nil { return nil, err } - isCN := loadParamFromVars("isCN", vars) - ctx := context.Background() - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc := oauth2.NewClient(ctx, ts) - client := odsdk.NewClient(tc) - if isCN == "true" { - client.BaseURL, _ = url.Parse("https://microsoftgraph.chinacloudapi.cn/v1.0/") + baseURL := oneDriveGlobalBaseURL + if loadParamFromVars("isCN", vars) == "true" { + baseURL = oneDriveChinaBaseURL } - return &oneDriveClient{client: *client}, nil -} - -func (o oneDriveClient) ListBuckets() ([]interface{}, error) { - return nil, nil -} - -func (o oneDriveClient) Exist(path string) (bool, error) { - path = "/" + strings.TrimPrefix(path, "/") - fileID, err := o.loadIDByPath(path) + parsedBaseURL, err := url.Parse(baseURL) if err != nil { - return false, err + return nil, fmt.Errorf("parse OneDrive base URL failed: %w", err) } - - return len(fileID) != 0, nil + return &oneDriveClient{client: http.DefaultClient, baseURL: parsedBaseURL, token: token}, nil } -func (o oneDriveClient) Size(path string) (int64, error) { - path = "/" + strings.TrimPrefix(path, "/") - pathItem := "root:" + path - if path == "/" { - pathItem = "root" - } - req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil) - if err != nil { - return 0, fmt.Errorf("new request for file id failed, err: %v", err) - } - var driveItem myDriverItem - if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil { - return 0, fmt.Errorf("do request for file id failed, err: %v", err) - } +func (o oneDriveClient) ListBuckets() ([]interface{}, error) { return nil, nil } - return driveItem.Size, nil +func (o oneDriveClient) Exist(itemPath string) (bool, error) { + _, err := o.loadIDByPath(normalizeDrivePath(itemPath)) + if isOneDriveNotFound(err) { + return false, nil + } + return err == nil, err } -type myDriverItem struct { - Name string `json:"name"` - Id string `json:"id"` - Size int64 `json:"size"` +func (o oneDriveClient) Size(itemPath string) (int64, error) { + var item DriveItem + if err := o.getDriveItem(context.Background(), normalizeDrivePath(itemPath), &item); err != nil { + return 0, err + } + return item.Size, nil } -func (o oneDriveClient) Delete(path string) (bool, error) { - path = "/" + strings.TrimPrefix(path, "/") - req, err := o.client.NewRequest("DELETE", fmt.Sprintf("me/drive/root:%s", path), nil) - if err != nil { - return false, fmt.Errorf("new request for delete file failed, err: %v \n", err) - } - if err := o.client.Do(context.Background(), req, false, nil); err != nil { - return false, fmt.Errorf("do request for delete file failed, err: %v \n", err) +func (o oneDriveClient) Delete(itemPath string) (bool, error) { + itemPath = normalizeDrivePath(itemPath) + if err := o.doJSON(context.Background(), http.MethodDelete, "me/drive/root:"+escapeDrivePath(itemPath), nil, nil); err != nil { + return false, fmt.Errorf("delete OneDrive file failed: %w", err) } - return true, nil } func (o oneDriveClient) Upload(ctx context.Context, src, target string) (bool, error) { - target = "/" + strings.TrimPrefix(target, "/") - if _, err := o.loadIDByPath(path.Dir(target)); err != nil { - if !strings.Contains(err.Error(), "itemNotFound") { + target = normalizeDrivePath(target) + parentPath := path.Dir(target) + if _, err := o.loadIDByPath(parentPath); err != nil { + if !isOneDriveNotFound(err) { return false, err } - if err := o.createFolder(path.Dir(target)); err != nil { - return false, fmt.Errorf("create dir before upload failed, err: %v", err) + if err := o.createFolder(parentPath); err != nil { + return false, fmt.Errorf("create directory before upload failed: %w", err) } } - - folderID, err := o.loadIDByPath(path.Dir(target)) + folderID, err := o.loadIDByPath(parentPath) if err != nil { return false, err } @@ -117,291 +92,312 @@ func (o oneDriveClient) Upload(ctx context.Context, src, target string) (bool, e if fileInfo.IsDir() { return false, errors.New("only file is allowed to be uploaded here") } - var isOk bool if fileInfo.Size() < 4*1024*1024 { - isOk, err = o.upSmall(ctx, src, folderID, fileInfo.Size()) - } else { - isOk, err = o.upBig(ctx, src, folderID, fileInfo.Size()) + return o.upSmall(ctx, src, folderID) } - return isOk, err + return o.upBig(ctx, src, folderID, fileInfo.Size()) } func (o oneDriveClient) Download(src, target string) (bool, error) { - src = "/" + strings.TrimPrefix(src, "/") - req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/root:%s", src), nil) - if err != nil { - return false, fmt.Errorf("new request for file id failed, err: %v", err) + var item DriveItem + if err := o.getDriveItem(context.Background(), normalizeDrivePath(src), &item); err != nil { + return false, err } - var driveItem *odsdk.DriveItem - if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil { - return false, fmt.Errorf("do request for file id failed, err: %v", err) + if item.DownloadURL == "" { + return false, errors.New("OneDrive download URL is missing") } - - resp, err := http.Get(driveItem.DownloadURL) + resp, err := o.client.Get(item.DownloadURL) if err != nil { return false, err } defer resp.Body.Close() - + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return false, newOneDriveHTTPError(resp) + } out, err := os.Create(target) if err != nil { return false, err } defer out.Close() - buffer := make([]byte, 2*1024*1024) - - _, err = io.CopyBuffer(out, resp.Body, buffer) - if err != nil { - return false, err - } - - return true, nil + _, err = io.CopyBuffer(out, resp.Body, make([]byte, 2*1024*1024)) + return err == nil, err } func (o *oneDriveClient) ListObjects(prefix string) ([]string, error) { - prefix = "/" + strings.TrimPrefix(prefix, "/") - folderID, err := o.loadIDByPath(prefix) + folderID, err := o.loadIDByPath(normalizeDrivePath(prefix)) if err != nil { return nil, err } - - req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/items/%s/children", folderID), nil) - if err != nil { - return nil, fmt.Errorf("new request for list failed, err: %v", err) + var items oneDriveItemsResponse + endpoint := fmt.Sprintf("me/drive/items/%s/children", url.PathEscape(folderID)) + if err := o.doJSON(context.Background(), http.MethodGet, endpoint, nil, &items); err != nil { + return nil, fmt.Errorf("list OneDrive files failed: %w", err) } - var driveItems *odsdk.OneDriveDriveItemsResponse - if err := o.client.Do(context.Background(), req, false, &driveItems); err != nil { - return nil, fmt.Errorf("do request for list failed, err: %v", err) + result := make([]string, 0, len(items.Value)) + for _, item := range items.Value { + result = append(result, item.Name) } + return result, nil +} - var itemList []string - for _, item := range driveItems.DriveItems { - itemList = append(itemList, item.Name) +func (o *oneDriveClient) loadIDByPath(itemPath string) (string, error) { + var item DriveItem + if err := o.getDriveItem(context.Background(), itemPath, &item); err != nil { + return "", err } - return itemList, nil + return item.ID, nil } -func (o *oneDriveClient) loadIDByPath(path string) (string, error) { - pathItem := "root:" + path - if path == "/" { - pathItem = "root" +func (o *oneDriveClient) getDriveItem(ctx context.Context, itemPath string, result *DriveItem) error { + endpoint := "me/drive/root" + if itemPath != "/" { + endpoint += ":" + escapeDrivePath(itemPath) } - req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil) - if err != nil { - return "", fmt.Errorf("new request for file id failed, err: %v", err) - } - var driveItem *odsdk.DriveItem - if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil { - return "", fmt.Errorf("do request for file id failed, err: %v", err) + if err := o.doJSON(ctx, http.MethodGet, endpoint, nil, result); err != nil { + return fmt.Errorf("get OneDrive item failed: %w", err) } - return driveItem.Id, nil + return nil } -func RefreshToken(grantType string, tokenType string, varMap map[string]interface{}) (string, error) { - data := url.Values{} - isCN := loadParamFromVars("isCN", varMap) - data.Set("client_id", loadParamFromVars("client_id", varMap)) - data.Set("client_secret", loadParamFromVars("client_secret", varMap)) - if grantType == "refresh_token" { - data.Set("grant_type", "refresh_token") - data.Set("refresh_token", loadParamFromVars("refresh_token", varMap)) - } else { - data.Set("grant_type", "authorization_code") - data.Set("code", loadParamFromVars("code", varMap)) - } - data.Set("redirect_uri", loadParamFromVars("redirect_uri", varMap)) - client := &http.Client{} - defer client.CloseIdleConnections() - url := "https://login.microsoftonline.com/common/oauth2/v2.0/token" - if isCN == "true" { - url = "https://login.chinacloudapi.cn/common/oauth2/v2.0/token" +func (o *oneDriveClient) createFolder(parent string) error { + if parent == "/" { + return nil } - req, err := http.NewRequest("POST", url, strings.NewReader(data.Encode())) + parentID, err := o.loadIDByPath(path.Dir(parent)) if err != nil { - return "", fmt.Errorf("new http post client for access token failed, err: %v", err) + if !isOneDriveNotFound(err) { + return err + } + if err := o.createFolder(path.Dir(parent)); err != nil { + return err + } + parentID, err = o.loadIDByPath(path.Dir(parent)) + if err != nil { + return err + } } - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - resp, err := client.Do(req) + body := struct { + Name string `json:"name"` + Folder map[string]interface{} `json:"folder"` + }{Name: path.Base(parent), Folder: map[string]interface{}{}} + endpoint := fmt.Sprintf("me/drive/items/%s/children", url.PathEscape(parentID)) + return o.doJSON(context.Background(), http.MethodPost, endpoint, body, nil) +} + +func (o *oneDriveClient) upSmall(ctx context.Context, srcPath, folderID string) (bool, error) { + file, err := os.Open(srcPath) if err != nil { - return "", fmt.Errorf("request for access token failed, err: %v", err) + return false, err } - defer resp.Body.Close() - respBody, err := io.ReadAll(resp.Body) + defer file.Close() + endpoint := fmt.Sprintf("me/drive/items/%s:/%s:/content?@microsoft.graph.conflictBehavior=rename", url.PathEscape(folderID), url.PathEscape(path.Base(srcPath))) + req, err := o.newGraphRequest(ctx, http.MethodPut, endpoint, file) if err != nil { - return "", fmt.Errorf("read data from response body failed, err: %v", err) + return false, err + } + req.Header.Set("Content-Type", files.GetMimeType(srcPath)) + if err := o.do(req, nil); err != nil { + return false, fmt.Errorf("upload OneDrive file failed: %w", err) } + return true, nil +} - tokenMap := map[string]interface{}{} - if err := json.Unmarshal(respBody, &tokenMap); err != nil { - return "", fmt.Errorf("unmarshal data from response body failed, err: %v", err) +func (o *oneDriveClient) upBig(ctx context.Context, srcPath, folderID string, fileSize int64) (bool, error) { + file, err := os.Open(srcPath) + if err != nil { + return false, err } - if tokenType == "accessToken" { - accessToken, ok := tokenMap["access_token"].(string) - if !ok { - return "", errors.New("no such access token in response") + defer file.Close() + body := struct { + Item struct { + ConflictBehavior string `json:"@microsoft.graph.conflictBehavior"` + } `json:"item"` + }{} + body.Item.ConflictBehavior = "rename" + var session oneDriveUploadSession + endpoint := fmt.Sprintf("me/drive/items/%s:/%s:/createUploadSession", url.PathEscape(folderID), url.PathEscape(path.Base(srcPath))) + if err := o.doJSON(ctx, http.MethodPost, endpoint, body, &session); err != nil { + return false, fmt.Errorf("create OneDrive upload session failed: %w", err) + } + + const chunkSize int64 = 5 * 1024 * 1024 + reader := bufio.NewReader(file) + buffer := make([]byte, chunkSize) + for offset := int64(0); offset < fileSize; { + length, readErr := io.ReadFull(reader, buffer) + if readErr != nil && !errors.Is(readErr, io.ErrUnexpectedEOF) && !errors.Is(readErr, io.EOF) { + return false, readErr } - tokenMap = nil - return accessToken, nil - } - refreshToken, ok := tokenMap["refresh_token"].(string) - if !ok { - return "", errors.New("no such access token in response") + if length == 0 { + return false, io.ErrUnexpectedEOF + } + if err := o.uploadChunk(ctx, session.UploadURL, offset, fileSize, buffer[:length]); err != nil { + return false, err + } + offset += int64(length) } - tokenMap = nil - return refreshToken, nil + return true, nil } -func (o *oneDriveClient) createFolder(parent string) error { - if _, err := o.loadIDByPath(path.Dir(parent)); err != nil { - if !strings.Contains(err.Error(), "itemNotFound") { - return err - } - _ = o.createFolder(path.Dir(parent)) - } - item2, err := o.loadIDByPath(path.Dir(parent)) +func (o *oneDriveClient) uploadChunk(ctx context.Context, uploadURL string, offset, total int64, chunk []byte) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(chunk)) if err != nil { return err } - if _, err := o.client.DriveItems.CreateNewFolder(context.Background(), "", item2, path.Base(parent)); err != nil { + req.Header.Set("Content-Length", strconv.Itoa(len(chunk))) + req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, offset+int64(len(chunk))-1, total)) + resp, err := o.client.Do(req) + if err != nil { return err } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusAccepted { + return newOneDriveHTTPError(resp) + } return nil } -type NewUploadSessionCreationRequest struct { - ConflictBehavior string `json:"@microsoft.graph.conflictBehavior,omitempty"` -} -type NewUploadSessionCreationResponse struct { - UploadURL string `json:"uploadUrl"` - ExpirationDateTime string `json:"expirationDateTime"` -} -type UploadSessionUploadResponse struct { - ExpirationDateTime string `json:"expirationDateTime"` - NextExpectedRanges []string `json:"nextExpectedRanges"` - DriveItem -} -type DriveItem struct { - Name string `json:"name"` - Id string `json:"id"` - DownloadURL string `json:"@microsoft.graph.downloadUrl"` - Description string `json:"description"` - Size int64 `json:"size"` - WebURL string `json:"webUrl"` +func (o *oneDriveClient) doJSON(ctx context.Context, method, endpoint string, body, result interface{}) error { + var reader io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return err + } + reader = bytes.NewReader(data) + } + req, err := o.newGraphRequest(ctx, method, endpoint, reader) + if err != nil { + return err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return o.do(req, result) } -func (o *oneDriveClient) NewSessionFileUploadRequest(ctx context.Context, absoluteUrl string, grandOffset, grandTotalSize int64, byteReader *bytes.Reader) (*http.Request, error) { - apiUrl, err := o.client.BaseURL.Parse(absoluteUrl) +func (o *oneDriveClient) newGraphRequest(ctx context.Context, method, endpoint string, body io.Reader) (*http.Request, error) { + apiURL, err := o.baseURL.Parse(endpoint) if err != nil { return nil, err } - absoluteUrl = apiUrl.String() - contentLength := byteReader.Size() - req, err := http.NewRequestWithContext(ctx, http.MethodPut, absoluteUrl, byteReader) + req, err := http.NewRequestWithContext(ctx, method, apiURL.String(), body) if err != nil { return nil, err } - req.Header.Set("Content-Length", strconv.FormatInt(contentLength, 10)) - preliminaryLength := grandOffset - preliminaryRange := grandOffset + contentLength - 1 - if preliminaryRange >= grandTotalSize { - preliminaryRange = grandTotalSize - 1 - preliminaryLength = preliminaryRange - grandOffset + 1 - } - req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", preliminaryLength, preliminaryRange, grandTotalSize)) - - return req, err + req.Header.Set("Authorization", "Bearer "+o.token) + return req, nil } -func (o *oneDriveClient) upSmall(ctx context.Context, srcPath, folderID string, fileSize int64) (bool, error) { - file, err := os.Open(srcPath) +func (o *oneDriveClient) do(req *http.Request, result interface{}) error { + resp, err := o.client.Do(req) if err != nil { - return false, err + return err } - defer file.Close() - - buffer := make([]byte, fileSize) - _, _ = file.Read(buffer) - fileReader := bytes.NewReader(buffer) - apiURL := fmt.Sprintf("me/drive/items/%s:/%s:/content?@microsoft.graph.conflictBehavior=rename", url.PathEscape(folderID), path.Base(srcPath)) - - mimeType := files.GetMimeType(srcPath) - req, err := o.client.NewFileUploadRequest(apiURL, mimeType, fileReader) - if err != nil { - return false, err + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return newOneDriveHTTPError(resp) } - var response *DriveItem - if err := o.client.Do(ctx, req, false, &response); err != nil { - return false, fmt.Errorf("do request for list failed, err: %v", err) + if result == nil || resp.StatusCode == http.StatusNoContent { + return nil } - return true, nil + return json.NewDecoder(resp.Body).Decode(result) } -func (o *oneDriveClient) upBig(ctx context.Context, srcPath, folderID string, fileSize int64) (bool, error) { - file, err := os.Open(srcPath) - if err != nil { - return false, err +type DriveItem struct { + Name string `json:"name"` + ID string `json:"id"` + DownloadURL string `json:"@microsoft.graph.downloadUrl"` + Size int64 `json:"size"` +} + +type oneDriveItemsResponse struct { + Value []DriveItem `json:"value"` +} + +type oneDriveUploadSession struct { + UploadURL string `json:"uploadUrl"` +} + +type oneDriveError struct { + Details struct { + Code string `json:"code"` + Message string `json:"message"` + } `json:"error"` + statusCode int +} + +func (e *oneDriveError) Error() string { + if e.Details.Code != "" || e.Details.Message != "" { + return fmt.Sprintf("OneDrive API error: %s: %s", e.Details.Code, e.Details.Message) } - defer file.Close() + return fmt.Sprintf("OneDrive API returned HTTP %d", e.statusCode) +} - apiURL := fmt.Sprintf("me/drive/items/%s:/%s:/createUploadSession", url.PathEscape(folderID), path.Base(srcPath)) - sessionCreationRequestInside := NewUploadSessionCreationRequest{ - ConflictBehavior: "rename", +func newOneDriveHTTPError(resp *http.Response) error { + apiErr := &oneDriveError{statusCode: resp.StatusCode} + if err := json.NewDecoder(resp.Body).Decode(apiErr); err != nil { + return fmt.Errorf("OneDrive API returned HTTP %d", resp.StatusCode) } + return apiErr +} - sessionCreationRequest := struct { - Item NewUploadSessionCreationRequest `json:"item"` - DeferCommit bool `json:"deferCommit"` - }{sessionCreationRequestInside, false} +func isOneDriveNotFound(err error) bool { + var apiErr *oneDriveError + return errors.As(err, &apiErr) && (apiErr.statusCode == http.StatusNotFound || apiErr.Details.Code == "itemNotFound") +} - sessionCreationReq, err := o.client.NewRequest("POST", apiURL, sessionCreationRequest) - if err != nil { - return false, err +func normalizeDrivePath(itemPath string) string { return "/" + strings.TrimPrefix(itemPath, "/") } + +func escapeDrivePath(itemPath string) string { + parts := strings.Split(strings.TrimPrefix(itemPath, "/"), "/") + for i, part := range parts { + parts[i] = url.PathEscape(part) } + return "/" + strings.Join(parts, "/") +} - var sessionCreationResp *NewUploadSessionCreationResponse - err = o.client.Do(ctx, sessionCreationReq, false, &sessionCreationResp) +func RefreshToken(grantType string, tokenType string, varMap map[string]interface{}) (string, error) { + data := url.Values{} + isCN := loadParamFromVars("isCN", varMap) + data.Set("client_id", loadParamFromVars("client_id", varMap)) + data.Set("client_secret", loadParamFromVars("client_secret", varMap)) + if grantType == "refresh_token" { + data.Set("grant_type", "refresh_token") + data.Set("refresh_token", loadParamFromVars("refresh_token", varMap)) + } else { + data.Set("grant_type", "authorization_code") + data.Set("code", loadParamFromVars("code", varMap)) + } + data.Set("redirect_uri", loadParamFromVars("redirect_uri", varMap)) + tokenURL := "https://login.microsoftonline.com/common/oauth2/v2.0/token" + if isCN == "true" { + tokenURL = "https://login.chinacloudapi.cn/common/oauth2/v2.0/token" + } + req, err := http.NewRequest(http.MethodPost, tokenURL, strings.NewReader(data.Encode())) if err != nil { - return false, fmt.Errorf("session creation failed %w", err) + return "", fmt.Errorf("create access token request failed: %w", err) } - - fileSessionUploadUrl := sessionCreationResp.UploadURL - - sizePerSplit := int64(5 * 1024 * 1024) - buffer := make([]byte, 5*1024*1024) - splitCount := fileSize / sizePerSplit - if fileSize%sizePerSplit != 0 { - splitCount += 1 + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("request access token failed: %w", err) } - bfReader := bufio.NewReader(file) - httpClient := http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return "", newOneDriveHTTPError(resp) } - for splitNow := int64(0); splitNow < splitCount; splitNow++ { - length, err := bfReader.Read(buffer) - if err != nil { - return false, err - } - if int64(length) < sizePerSplit { - bufferLast := buffer[:length] - buffer = bufferLast - } - sessionFileUploadReq, err := o.NewSessionFileUploadRequest(ctx, fileSessionUploadUrl, splitNow*sizePerSplit, fileSize, bytes.NewReader(buffer)) - if err != nil { - return false, err - } - res, err := httpClient.Do(sessionFileUploadReq) - if err != nil { - return false, err - } - if res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200 { - data, _ := io.ReadAll(res.Body) - res.Body.Close() - return false, errors.New(string(data)) - } - res.Body.Close() + var tokenMap map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&tokenMap); err != nil { + return "", fmt.Errorf("decode token response failed: %w", err) } - return true, nil + key := "refresh_token" + if tokenType == "accessToken" { + key = "access_token" + } + token, ok := tokenMap[key].(string) + if !ok { + return "", fmt.Errorf("no %s in token response", key) + } + return token, nil } diff --git a/frontend/src/views/cronjob/cronjob/index.vue b/frontend/src/views/cronjob/cronjob/index.vue index 7d56e7d3738d..180efeee89a2 100644 --- a/frontend/src/views/cronjob/cronjob/index.vue +++ b/frontend/src/views/cronjob/cronjob/index.vue @@ -202,11 +202,11 @@