Clone/aliyunpan
1
0
Fork 0
mirror of https://github.com/tickstep/aliyunpan.git synced 2025-01-23 14:32:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

use openapi for upload commnad

Browse Source
This commit is contained in:
tickstep 2024-03-02 22:58:41 +08:00
parent 3bdc27ce66
commit 69b76a15ca
5 changed files with 60 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
internal/command/download.go
View File

@ -25,15 +25,12 @@ import (
"github.com/tickstep/aliyunpan/internal/utils"
"github.com/tickstep/aliyunpan/library/requester/transfer"
"github.com/tickstep/library-go/converter"
"github.com/tickstep/library-go/logger"
"github.com/tickstep/library-go/requester/rio/speeds"
"github.com/urfave/cli"
"os"
"path"
"path/filepath"
"runtime"
"sync/atomic"
"time"
)
type (
@ -222,20 +219,21 @@ func RunDownload(paths []string, options *DownloadOptions) {
activeUser.PanClient().OpenapiPanClient().EnableCache()
activeUser.PanClient().OpenapiPanClient().ClearCache()
defer activeUser.PanClient().OpenapiPanClient().DisableCache()
// pan token expired checker
continueFlag := int32(0)
atomic.StoreInt32(&continueFlag, 0)
defer func() {
atomic.StoreInt32(&continueFlag, 1)
}()
go func(flag *int32) {
for atomic.LoadInt32(flag) == 0 {
time.Sleep(time.Duration(1) * time.Minute)
if RefreshWebTokenInNeed(activeUser, config.Config.DeviceName) {
logger.Verboseln("update access token for download task")
}
}
}(&continueFlag)
//// pan token expired checker
//continueFlag := int32(0)
//atomic.StoreInt32(&continueFlag, 0)
//defer func() {
// atomic.StoreInt32(&continueFlag, 1)
//}()
//go func(flag *int32) {
// for atomic.LoadInt32(flag) == 0 {
// time.Sleep(time.Duration(1) * time.Minute)
// if RefreshWebTokenInNeed(activeUser, config.Config.DeviceName) {
// logger.Verboseln("update access token for download task")
// }
// }
//}(&continueFlag)
if options == nil {
options = &DownloadOptions{}

46
internal/command/upload.go
View File

@ -26,7 +26,6 @@ import (
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/tickstep/library-go/logger"
@ -220,23 +219,24 @@ func CmdUpload() cli.Command {
// RunUpload 执行文件上传
func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
activeUser := GetActiveUser()
activeUser.PanClient().WebapiPanClient().EnableCache()
activeUser.PanClient().WebapiPanClient().ClearCache()
defer activeUser.PanClient().WebapiPanClient().DisableCache()
// pan token expired checker
continueFlag := int32(0)
atomic.StoreInt32(&continueFlag, 0)
defer func() {
atomic.StoreInt32(&continueFlag, 1)
}()
go func(flag *int32) {
for atomic.LoadInt32(flag) == 0 {
time.Sleep(time.Duration(1) * time.Minute)
if RefreshWebTokenInNeed(activeUser, config.Config.DeviceName) {
logger.Verboseln("update access token for upload task")
}
}
}(&continueFlag)
activeUser.PanClient().OpenapiPanClient().EnableCache()
activeUser.PanClient().OpenapiPanClient().ClearCache()
defer activeUser.PanClient().OpenapiPanClient().DisableCache()
//// pan token expired checker
//continueFlag := int32(0)
//atomic.StoreInt32(&continueFlag, 0)
//defer func() {
// atomic.StoreInt32(&continueFlag, 1)
//}()
//go func(flag *int32) {
// for atomic.LoadInt32(flag) == 0 {
// time.Sleep(time.Duration(1) * time.Minute)
// if RefreshWebTokenInNeed(activeUser, config.Config.DeviceName) {
// logger.Verboseln("update access token for upload task")
// }
// }
//}(&continueFlag)
if opt == nil {
opt = &UploadOptions{}
@ -263,13 +263,13 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
// 超时时间
if opt.MaxTimeoutSec > 0 {
activeUser.PanClient().WebapiPanClient().SetTimeout(time.Duration(opt.MaxTimeoutSec) * time.Second)
activeUser.PanClient().OpenapiPanClient().SetTimeout(time.Duration(opt.MaxTimeoutSec) * time.Second)
}
fmt.Printf("\n[0] 当前文件上传最大并发量为: %d, 上传分片大小为: %s\n", opt.AllParallel, converter.ConvertFileSize(opt.BlockSize, 2))
savePath = activeUser.PathJoin(opt.DriveId, savePath)
_, err1 := activeUser.PanClient().WebapiPanClient().FileInfoByPath(opt.DriveId, savePath)
_, err1 := activeUser.PanClient().OpenapiPanClient().FileInfoByPath(opt.DriveId, savePath)
if err1 != nil {
fmt.Printf("警告: 上传文件, 获取云盘路径 %s 错误, %s\n", savePath, err1)
}
@ -388,7 +388,7 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
LocalFileChecksum: localfile.NewLocalSymlinkFileEntity(file),
SavePath: subSavePath,
DriveId: opt.DriveId,
PanClient: activeUser.PanClient().WebapiPanClient(),
PanClient: activeUser.PanClient(),
UploadingDatabase: uploadDatabase,
FolderCreateMutex: folderCreateMutex,
Parallel: opt.Parallel,
@ -410,11 +410,11 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
if saveFilePath != "/" {
folderCreateMutex.Lock()
fmt.Printf("正在检测和创建云盘文件夹: %s\n", saveFilePath)
_, apierr1 := activeUser.PanClient().WebapiPanClient().FileInfoByPath(opt.DriveId, saveFilePath)
_, apierr1 := activeUser.PanClient().OpenapiPanClient().FileInfoByPath(opt.DriveId, saveFilePath)
time.Sleep(1 * time.Second)
if apierr1 != nil && apierr1.Code == apierror.ApiCodeFileNotFoundCode {
logger.Verbosef("%s 创建云盘文件夹: %s\n", utils.NowTimeStr(), saveFilePath)
rs, apierr := activeUser.PanClient().WebapiPanClient().Mkdir(opt.DriveId, "root", saveFilePath)
rs, apierr := activeUser.PanClient().OpenapiPanClient().MkdirByFullPath(opt.DriveId, saveFilePath)
if apierr != nil || rs.FileId == "" {
fmt.Printf("创建云盘文件夹失败: %s\n", saveFilePath)
}

34
internal/functions/panupload/upload.go
View File

@ -4,7 +4,7 @@
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@ -16,6 +16,7 @@ package panupload
import (
"context"
"encoding/xml"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/library-go/logger"
"github.com/tickstep/library-go/requester"
"io"
@ -31,7 +32,7 @@ import (
type (
PanUpload struct {
panClient *aliyunpan.PanClient
panClient *config.PanClient
targetPath string
driveId string
@ -52,7 +53,7 @@ func (e EmptyReaderLen64) Len() int64 {
return 0
}
func NewPanUpload(panClient *aliyunpan.PanClient, targetPath, driveId string, uploadOpEntity *aliyunpan.CreateFileUploadResult, useInternalUrl bool) uploader.MultiUpload {
func NewPanUpload(panClient *config.PanClient, targetPath, driveId string, uploadOpEntity *aliyunpan.CreateFileUploadResult, useInternalUrl bool) uploader.MultiUpload {
return &PanUpload{
panClient: panClient,
targetPath: targetPath,
@ -64,7 +65,7 @@ func NewPanUpload(panClient *aliyunpan.PanClient, targetPath, driveId string, up
func (pu *PanUpload) lazyInit() {
if pu.panClient == nil {
pu.panClient = &aliyunpan.PanClient{}
pu.panClient = &config.PanClient{}
}
}
@ -94,7 +95,7 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
PartInfoList: infoList,
UploadId: pu.uploadOpEntity.UploadId,
}
newUploadInfo, err := pu.panClient.GetUploadUrl(refreshUploadParam)
newUploadInfo, err := pu.panClient.OpenapiPanClient().GetUploadUrl(refreshUploadParam)
if err != nil {
logger.Verboseln(err)
return false, &uploader.MultiError{
@ -185,12 +186,12 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
if pu.useInternalUrl {
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL
}
apiError := pu.panClient.UploadFileData(uploadUrl, uploadFunc)
apiError := pu.panClient.OpenapiPanClient().UploadFileData(uploadUrl, uploadFunc)
if respErr != nil {
if respErr.Err == uploader.UploadUrlExpired {
// URL过期，获取新的URL
guur, er := pu.panClient.GetUploadUrl(&aliyunpan.GetUploadUrlParam{
guur, er := pu.panClient.OpenapiPanClient().GetUploadUrl(&aliyunpan.GetUploadUrlParam{
DriveId: pu.driveId,
FileId: pu.uploadOpEntity.FileId,
UploadId: pu.uploadOpEntity.UploadId,
@ -208,7 +209,7 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
if pu.useInternalUrl {
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL
}
apiError = pu.panClient.UploadFileData(uploadUrl, uploadFunc)
apiError = pu.panClient.OpenapiPanClient().UploadFileData(uploadUrl, uploadFunc)
} else if respErr.Err == uploader.UploadPartAlreadyExist {
// already upload
// success
@ -233,24 +234,11 @@ func (pu *PanUpload) CommitFile() (cerr error) {
pu.lazyInit()
var er *apierror.ApiError
_, er = pu.panClient.CompleteUploadFile(&aliyunpan.CompleteUploadFileParam{
_, er = pu.panClient.OpenapiPanClient().CompleteUploadFile(&aliyunpan.CompleteUploadFileParam{
DriveId: pu.driveId,
FileId: pu.uploadOpEntity.FileId,
UploadId: pu.uploadOpEntity.UploadId,
})
if er != nil && er.Code == apierror.ApiCodeDeviceSessionSignatureInvalid {
_, e := pu.panClient.CreateSession(nil)
if e == nil {
// retry
_, er = pu.panClient.CompleteUploadFile(&aliyunpan.CompleteUploadFileParam{
DriveId: pu.driveId,
FileId: pu.uploadOpEntity.FileId,
UploadId: pu.uploadOpEntity.UploadId,
})
} else {
logger.Verboseln("CreateSession failed")
}
}
if er != nil {
return er
}
@ -266,7 +254,7 @@ func (pu *PanUpload) triggerVideoTranscodeAction() {
// 视频文件触发云端转码请求
if pu.uploadOpEntity != nil && IsVideoFile(pu.uploadOpEntity.FileName) {
time.Sleep(3 * time.Second)
_, er1 := pu.panClient.VideoGetPreviewPlayInfo(&aliyunpan.VideoGetPreviewPlayInfoParam{
_, er1 := pu.panClient.OpenapiPanClient().VideoGetPreviewPlayInfo(&aliyunpan.VideoGetPreviewPlayInfoParam{
DriveId: pu.driveId,
FileId: pu.uploadOpEntity.FileId,
})

34
internal/functions/panupload/upload_task_unit.go
View File

@ -4,7 +4,7 @@
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@ -51,7 +51,7 @@ type (
DriveId string // 网盘ID，例如：文件网盘，相册网盘
FolderCreateMutex *sync.Mutex
PanClient *aliyunpan.PanClient
PanClient *config.PanClient
UploadingDatabase *UploadingDatabase // 数据库
Parallel int
NoRapidUpload bool // 禁用秒传，无需计算SHA1，直接上传
@ -356,7 +356,7 @@ StepUploadPrepareUpload:
if saveFilePath != "/" {
utu.FolderCreateMutex.Lock()
fmt.Printf("[%s] %s 正在检测和创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath)
fe, apierr1 := utu.PanClient.FileInfoByPath(utu.DriveId, saveFilePath)
fe, apierr1 := utu.PanClient.OpenapiPanClient().FileInfoByPath(utu.DriveId, saveFilePath)
time.Sleep(1 * time.Second)
needToCreateFolder := false
if apierr1 != nil && apierr1.Code == apierror.ApiCodeFileNotFoundCode {
@ -371,16 +371,7 @@ StepUploadPrepareUpload:
}
if needToCreateFolder {
logger.Verbosef("[%s] %s 创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath)
rs, apierr = utu.PanClient.Mkdir(utu.DriveId, "root", saveFilePath)
if apierr != nil && apierr.Code == apierror.ApiCodeDeviceSessionSignatureInvalid {
_, e := utu.PanClient.CreateSession(nil)
if e == nil {
// retry
rs, apierr = utu.PanClient.Mkdir(utu.DriveId, "root", saveFilePath)
} else {
logger.Verboseln("CreateSession failed")
}
}
rs, apierr = utu.PanClient.OpenapiPanClient().MkdirByFullPath(utu.DriveId, saveFilePath)
if apierr != nil || rs.FileId == "" {
result.Err = apierr
result.ResultMessage = "创建云盘文件夹失败"
@ -402,7 +393,7 @@ StepUploadPrepareUpload:
checkNameMode = "auto_rename"
// 如果启用了 覆盖/跳过 已存在的文件,则需要提前检查文件是否存在
if utu.IsOverwrite || utu.IsSkipSameName {
efi, apierr = utu.PanClient.FileInfoByPath(utu.DriveId, utu.SavePath)
efi, apierr = utu.PanClient.OpenapiPanClient().FileInfoByPath(utu.DriveId, utu.SavePath)
if apierr != nil && apierr.Code != apierror.ApiCodeFileNotFoundCode {
result.Err = apierr
result.ResultMessage = "检测同名文件失败"
@ -429,7 +420,7 @@ StepUploadPrepareUpload:
// proof code
localFile, _ = os.Open(utu.LocalFileChecksum.Path.RealPath)
localFileInfo, _ = localFile.Stat()
proofCode = aliyunpan.CalcProofCode(utu.PanClient.GetAccessToken(), rio.NewFileReaderAtLen64(localFile), localFileInfo.Size())
proofCode = aliyunpan.CalcProofCode(utu.PanClient.OpenapiPanClient().GetAccessToken(), rio.NewFileReaderAtLen64(localFile), localFileInfo.Size())
localFile.Close()
} else {
fmt.Printf("[%s] %s 已经禁用秒传检测，直接上传\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"))
@ -450,7 +441,7 @@ StepUploadPrepareUpload:
// existed, delete it
var fileDeleteResult []*aliyunpan.FileBatchActionResult
var err *apierror.ApiError
fileDeleteResult, err = utu.PanClient.FileDelete([]*aliyunpan.FileBatchActionParam{{DriveId: efi.DriveId, FileId: efi.FileId}})
fileDeleteResult, err = utu.PanClient.OpenapiPanClient().FileDelete([]*aliyunpan.FileBatchActionParam{{DriveId: efi.DriveId, FileId: efi.FileId}})
if err != nil || len(fileDeleteResult) == 0 {
result.Err = err
result.ResultMessage = "无法删除文件，请稍后重试"
@ -482,16 +473,7 @@ StepUploadPrepareUpload:
ProofVersion: "v1",
}
uploadOpEntity, apierr = utu.PanClient.CreateUploadFile(appCreateUploadFileParam)
if apierr != nil && apierr.Code == apierror.ApiCodeDeviceSessionSignatureInvalid {
_, e := utu.PanClient.CreateSession(nil)
if e == nil {
// retry
uploadOpEntity, apierr = utu.PanClient.CreateUploadFile(appCreateUploadFileParam)
} else {
logger.Verboseln("CreateSession failed")
}
}
uploadOpEntity, apierr = utu.PanClient.OpenapiPanClient().CreateUploadFile(appCreateUploadFileParam)
if apierr != nil {
result.Err = apierr
result.ResultMessage = "创建上传任务失败：" + apierr.Error()

4
internal/syncdrive/file_action_task.go
View File

@ -569,7 +569,9 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
// 创建分片上传器
// 阿里云盘默认就是分片上传，每一个分片对应一个part_info
// 但是不支持分片同时上传，必须单线程，并且按照顺序从1开始一个一个上传
worker := panupload.NewPanUpload(f.panClient, f.syncItem.getPanFileFullPath(), f.syncItem.DriveId, f.syncItem.UploadEntity, f.syncItem.UseInternalUrl)
// TODO: need fix
//worker := panupload.NewPanUpload(f.panClient, f.syncItem.getPanFileFullPath(), f.syncItem.DriveId, f.syncItem.UploadEntity, f.syncItem.UseInternalUrl)
worker := panupload.NewPanUpload(nil, f.syncItem.getPanFileFullPath(), f.syncItem.DriveId, f.syncItem.UploadEntity, f.syncItem.UseInternalUrl)
// 限速配置
var rateLimit *speeds.RateLimit