mirror of
https://github.com/tickstep/aliyunpan.git
synced 2025-01-23 14:32:14 +08:00
use openapi for upload commnad
This commit is contained in:
tickstep
parent
3bdc27ce66
commit
69b76a15ca
@ -25,15 +25,12 @@ import (
|
||||
"github.com/tickstep/aliyunpan/internal/utils"
|
||||
"github.com/tickstep/aliyunpan/library/requester/transfer"
|
||||
"github.com/tickstep/library-go/converter"
|
||||
"github.com/tickstep/library-go/logger"
|
||||
"github.com/tickstep/library-go/requester/rio/speeds"
|
||||
"github.com/urfave/cli"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
type (
|
||||
@ -222,20 +219,21 @@ func RunDownload(paths []string, options *DownloadOptions) {
|
||||
activeUser.PanClient().OpenapiPanClient().EnableCache()
|
||||
activeUser.PanClient().OpenapiPanClient().ClearCache()
|
||||
defer activeUser.PanClient().OpenapiPanClient().DisableCache()
|
||||
// pan token expired checker
|
||||
continueFlag := int32(0)
|
||||
atomic.StoreInt32(&continueFlag, 0)
|
||||
defer func() {
|
||||
atomic.StoreInt32(&continueFlag, 1)
|
||||
}()
|
||||
go func(flag *int32) {
|
||||
for atomic.LoadInt32(flag) == 0 {
|
||||
time.Sleep(time.Duration(1) * time.Minute)
|
||||
if RefreshWebTokenInNeed(activeUser, config.Config.DeviceName) {
|
||||
logger.Verboseln("update access token for download task")
|
||||
}
|
||||
}
|
||||
}(&continueFlag)
|
||||
|
||||
//// pan token expired checker
|
||||
//continueFlag := int32(0)
|
||||
//atomic.StoreInt32(&continueFlag, 0)
|
||||
//defer func() {
|
||||
// atomic.StoreInt32(&continueFlag, 1)
|
||||
//}()
|
||||
//go func(flag *int32) {
|
||||
// for atomic.LoadInt32(flag) == 0 {
|
||||
// time.Sleep(time.Duration(1) * time.Minute)
|
||||
// if RefreshWebTokenInNeed(activeUser, config.Config.DeviceName) {
|
||||
// logger.Verboseln("update access token for download task")
|
||||
// }
|
||||
// }
|
||||
//}(&continueFlag)
|
||||
|
||||
if options == nil {
|
||||
options = &DownloadOptions{}
|
||||
|
||||
@ -26,7 +26,6 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/tickstep/library-go/logger"
|
||||
@ -220,23 +219,24 @@ func CmdUpload() cli.Command {
|
||||
// RunUpload 执行文件上传
|
||||
func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
|
||||
activeUser := GetActiveUser()
|
||||
activeUser.PanClient().WebapiPanClient().EnableCache()
|
||||
activeUser.PanClient().WebapiPanClient().ClearCache()
|
||||
defer activeUser.PanClient().WebapiPanClient().DisableCache()
|
||||
// pan token expired checker
|
||||
continueFlag := int32(0)
|
||||
atomic.StoreInt32(&continueFlag, 0)
|
||||
defer func() {
|
||||
atomic.StoreInt32(&continueFlag, 1)
|
||||
}()
|
||||
go func(flag *int32) {
|
||||
for atomic.LoadInt32(flag) == 0 {
|
||||
time.Sleep(time.Duration(1) * time.Minute)
|
||||
if RefreshWebTokenInNeed(activeUser, config.Config.DeviceName) {
|
||||
logger.Verboseln("update access token for upload task")
|
||||
}
|
||||
}
|
||||
}(&continueFlag)
|
||||
activeUser.PanClient().OpenapiPanClient().EnableCache()
|
||||
activeUser.PanClient().OpenapiPanClient().ClearCache()
|
||||
defer activeUser.PanClient().OpenapiPanClient().DisableCache()
|
||||
|
||||
//// pan token expired checker
|
||||
//continueFlag := int32(0)
|
||||
//atomic.StoreInt32(&continueFlag, 0)
|
||||
//defer func() {
|
||||
// atomic.StoreInt32(&continueFlag, 1)
|
||||
//}()
|
||||
//go func(flag *int32) {
|
||||
// for atomic.LoadInt32(flag) == 0 {
|
||||
// time.Sleep(time.Duration(1) * time.Minute)
|
||||
// if RefreshWebTokenInNeed(activeUser, config.Config.DeviceName) {
|
||||
// logger.Verboseln("update access token for upload task")
|
||||
// }
|
||||
// }
|
||||
//}(&continueFlag)
|
||||
|
||||
if opt == nil {
|
||||
opt = &UploadOptions{}
|
||||
@ -263,13 +263,13 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
|
||||
|
||||
// 超时时间
|
||||
if opt.MaxTimeoutSec > 0 {
|
||||
activeUser.PanClient().WebapiPanClient().SetTimeout(time.Duration(opt.MaxTimeoutSec) * time.Second)
|
||||
activeUser.PanClient().OpenapiPanClient().SetTimeout(time.Duration(opt.MaxTimeoutSec) * time.Second)
|
||||
}
|
||||
|
||||
fmt.Printf("\n[0] 当前文件上传最大并发量为: %d, 上传分片大小为: %s\n", opt.AllParallel, converter.ConvertFileSize(opt.BlockSize, 2))
|
||||
|
||||
savePath = activeUser.PathJoin(opt.DriveId, savePath)
|
||||
_, err1 := activeUser.PanClient().WebapiPanClient().FileInfoByPath(opt.DriveId, savePath)
|
||||
_, err1 := activeUser.PanClient().OpenapiPanClient().FileInfoByPath(opt.DriveId, savePath)
|
||||
if err1 != nil {
|
||||
fmt.Printf("警告: 上传文件, 获取云盘路径 %s 错误, %s\n", savePath, err1)
|
||||
}
|
||||
@ -388,7 +388,7 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
|
||||
LocalFileChecksum: localfile.NewLocalSymlinkFileEntity(file),
|
||||
SavePath: subSavePath,
|
||||
DriveId: opt.DriveId,
|
||||
PanClient: activeUser.PanClient().WebapiPanClient(),
|
||||
PanClient: activeUser.PanClient(),
|
||||
UploadingDatabase: uploadDatabase,
|
||||
FolderCreateMutex: folderCreateMutex,
|
||||
Parallel: opt.Parallel,
|
||||
@ -410,11 +410,11 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
|
||||
if saveFilePath != "/" {
|
||||
folderCreateMutex.Lock()
|
||||
fmt.Printf("正在检测和创建云盘文件夹: %s\n", saveFilePath)
|
||||
_, apierr1 := activeUser.PanClient().WebapiPanClient().FileInfoByPath(opt.DriveId, saveFilePath)
|
||||
_, apierr1 := activeUser.PanClient().OpenapiPanClient().FileInfoByPath(opt.DriveId, saveFilePath)
|
||||
time.Sleep(1 * time.Second)
|
||||
if apierr1 != nil && apierr1.Code == apierror.ApiCodeFileNotFoundCode {
|
||||
logger.Verbosef("%s 创建云盘文件夹: %s\n", utils.NowTimeStr(), saveFilePath)
|
||||
rs, apierr := activeUser.PanClient().WebapiPanClient().Mkdir(opt.DriveId, "root", saveFilePath)
|
||||
rs, apierr := activeUser.PanClient().OpenapiPanClient().MkdirByFullPath(opt.DriveId, saveFilePath)
|
||||
if apierr != nil || rs.FileId == "" {
|
||||
fmt.Printf("创建云盘文件夹失败: %s\n", saveFilePath)
|
||||
}
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@ -16,6 +16,7 @@ package panupload
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"github.com/tickstep/aliyunpan/internal/config"
|
||||
"github.com/tickstep/library-go/logger"
|
||||
"github.com/tickstep/library-go/requester"
|
||||
"io"
|
||||
@ -31,7 +32,7 @@ import (
|
||||
|
||||
type (
|
||||
PanUpload struct {
|
||||
panClient *aliyunpan.PanClient
|
||||
panClient *config.PanClient
|
||||
targetPath string
|
||||
driveId string
|
||||
|
||||
@ -52,7 +53,7 @@ func (e EmptyReaderLen64) Len() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func NewPanUpload(panClient *aliyunpan.PanClient, targetPath, driveId string, uploadOpEntity *aliyunpan.CreateFileUploadResult, useInternalUrl bool) uploader.MultiUpload {
|
||||
func NewPanUpload(panClient *config.PanClient, targetPath, driveId string, uploadOpEntity *aliyunpan.CreateFileUploadResult, useInternalUrl bool) uploader.MultiUpload {
|
||||
return &PanUpload{
|
||||
panClient: panClient,
|
||||
targetPath: targetPath,
|
||||
@ -64,7 +65,7 @@ func NewPanUpload(panClient *aliyunpan.PanClient, targetPath, driveId string, up
|
||||
|
||||
func (pu *PanUpload) lazyInit() {
|
||||
if pu.panClient == nil {
|
||||
pu.panClient = &aliyunpan.PanClient{}
|
||||
pu.panClient = &config.PanClient{}
|
||||
}
|
||||
}
|
||||
|
||||
@ -94,7 +95,7 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
|
||||
PartInfoList: infoList,
|
||||
UploadId: pu.uploadOpEntity.UploadId,
|
||||
}
|
||||
newUploadInfo, err := pu.panClient.GetUploadUrl(refreshUploadParam)
|
||||
newUploadInfo, err := pu.panClient.OpenapiPanClient().GetUploadUrl(refreshUploadParam)
|
||||
if err != nil {
|
||||
logger.Verboseln(err)
|
||||
return false, &uploader.MultiError{
|
||||
@ -185,12 +186,12 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
|
||||
if pu.useInternalUrl {
|
||||
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL
|
||||
}
|
||||
apiError := pu.panClient.UploadFileData(uploadUrl, uploadFunc)
|
||||
apiError := pu.panClient.OpenapiPanClient().UploadFileData(uploadUrl, uploadFunc)
|
||||
|
||||
if respErr != nil {
|
||||
if respErr.Err == uploader.UploadUrlExpired {
|
||||
// URL过期，获取新的URL
|
||||
guur, er := pu.panClient.GetUploadUrl(&aliyunpan.GetUploadUrlParam{
|
||||
guur, er := pu.panClient.OpenapiPanClient().GetUploadUrl(&aliyunpan.GetUploadUrlParam{
|
||||
DriveId: pu.driveId,
|
||||
FileId: pu.uploadOpEntity.FileId,
|
||||
UploadId: pu.uploadOpEntity.UploadId,
|
||||
@ -208,7 +209,7 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
|
||||
if pu.useInternalUrl {
|
||||
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL
|
||||
}
|
||||
apiError = pu.panClient.UploadFileData(uploadUrl, uploadFunc)
|
||||
apiError = pu.panClient.OpenapiPanClient().UploadFileData(uploadUrl, uploadFunc)
|
||||
} else if respErr.Err == uploader.UploadPartAlreadyExist {
|
||||
// already upload
|
||||
// success
|
||||
@ -233,24 +234,11 @@ func (pu *PanUpload) CommitFile() (cerr error) {
|
||||
pu.lazyInit()
|
||||
var er *apierror.ApiError
|
||||
|
||||
_, er = pu.panClient.CompleteUploadFile(&aliyunpan.CompleteUploadFileParam{
|
||||
_, er = pu.panClient.OpenapiPanClient().CompleteUploadFile(&aliyunpan.CompleteUploadFileParam{
|
||||
DriveId: pu.driveId,
|
||||
FileId: pu.uploadOpEntity.FileId,
|
||||
UploadId: pu.uploadOpEntity.UploadId,
|
||||
})
|
||||
if er != nil && er.Code == apierror.ApiCodeDeviceSessionSignatureInvalid {
|
||||
_, e := pu.panClient.CreateSession(nil)
|
||||
if e == nil {
|
||||
// retry
|
||||
_, er = pu.panClient.CompleteUploadFile(&aliyunpan.CompleteUploadFileParam{
|
||||
DriveId: pu.driveId,
|
||||
FileId: pu.uploadOpEntity.FileId,
|
||||
UploadId: pu.uploadOpEntity.UploadId,
|
||||
})
|
||||
} else {
|
||||
logger.Verboseln("CreateSession failed")
|
||||
}
|
||||
}
|
||||
if er != nil {
|
||||
return er
|
||||
}
|
||||
@ -266,7 +254,7 @@ func (pu *PanUpload) triggerVideoTranscodeAction() {
|
||||
// 视频文件触发云端转码请求
|
||||
if pu.uploadOpEntity != nil && IsVideoFile(pu.uploadOpEntity.FileName) {
|
||||
time.Sleep(3 * time.Second)
|
||||
_, er1 := pu.panClient.VideoGetPreviewPlayInfo(&aliyunpan.VideoGetPreviewPlayInfoParam{
|
||||
_, er1 := pu.panClient.OpenapiPanClient().VideoGetPreviewPlayInfo(&aliyunpan.VideoGetPreviewPlayInfoParam{
|
||||
DriveId: pu.driveId,
|
||||
FileId: pu.uploadOpEntity.FileId,
|
||||
})
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@ -51,7 +51,7 @@ type (
|
||||
DriveId string // 网盘ID，例如：文件网盘，相册网盘
|
||||
FolderCreateMutex *sync.Mutex
|
||||
|
||||
PanClient *aliyunpan.PanClient
|
||||
PanClient *config.PanClient
|
||||
UploadingDatabase *UploadingDatabase // 数据库
|
||||
Parallel int
|
||||
NoRapidUpload bool // 禁用秒传，无需计算SHA1，直接上传
|
||||
@ -356,7 +356,7 @@ StepUploadPrepareUpload:
|
||||
if saveFilePath != "/" {
|
||||
utu.FolderCreateMutex.Lock()
|
||||
fmt.Printf("[%s] %s 正在检测和创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath)
|
||||
fe, apierr1 := utu.PanClient.FileInfoByPath(utu.DriveId, saveFilePath)
|
||||
fe, apierr1 := utu.PanClient.OpenapiPanClient().FileInfoByPath(utu.DriveId, saveFilePath)
|
||||
time.Sleep(1 * time.Second)
|
||||
needToCreateFolder := false
|
||||
if apierr1 != nil && apierr1.Code == apierror.ApiCodeFileNotFoundCode {
|
||||
@ -371,16 +371,7 @@ StepUploadPrepareUpload:
|
||||
}
|
||||
if needToCreateFolder {
|
||||
logger.Verbosef("[%s] %s 创建云盘文件夹: %s\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"), saveFilePath)
|
||||
rs, apierr = utu.PanClient.Mkdir(utu.DriveId, "root", saveFilePath)
|
||||
if apierr != nil && apierr.Code == apierror.ApiCodeDeviceSessionSignatureInvalid {
|
||||
_, e := utu.PanClient.CreateSession(nil)
|
||||
if e == nil {
|
||||
// retry
|
||||
rs, apierr = utu.PanClient.Mkdir(utu.DriveId, "root", saveFilePath)
|
||||
} else {
|
||||
logger.Verboseln("CreateSession failed")
|
||||
}
|
||||
}
|
||||
rs, apierr = utu.PanClient.OpenapiPanClient().MkdirByFullPath(utu.DriveId, saveFilePath)
|
||||
if apierr != nil || rs.FileId == "" {
|
||||
result.Err = apierr
|
||||
result.ResultMessage = "创建云盘文件夹失败"
|
||||
@ -402,7 +393,7 @@ StepUploadPrepareUpload:
|
||||
checkNameMode = "auto_rename"
|
||||
// 如果启用了 覆盖/跳过 已存在的文件,则需要提前检查文件是否存在
|
||||
if utu.IsOverwrite || utu.IsSkipSameName {
|
||||
efi, apierr = utu.PanClient.FileInfoByPath(utu.DriveId, utu.SavePath)
|
||||
efi, apierr = utu.PanClient.OpenapiPanClient().FileInfoByPath(utu.DriveId, utu.SavePath)
|
||||
if apierr != nil && apierr.Code != apierror.ApiCodeFileNotFoundCode {
|
||||
result.Err = apierr
|
||||
result.ResultMessage = "检测同名文件失败"
|
||||
@ -429,7 +420,7 @@ StepUploadPrepareUpload:
|
||||
// proof code
|
||||
localFile, _ = os.Open(utu.LocalFileChecksum.Path.RealPath)
|
||||
localFileInfo, _ = localFile.Stat()
|
||||
proofCode = aliyunpan.CalcProofCode(utu.PanClient.GetAccessToken(), rio.NewFileReaderAtLen64(localFile), localFileInfo.Size())
|
||||
proofCode = aliyunpan.CalcProofCode(utu.PanClient.OpenapiPanClient().GetAccessToken(), rio.NewFileReaderAtLen64(localFile), localFileInfo.Size())
|
||||
localFile.Close()
|
||||
} else {
|
||||
fmt.Printf("[%s] %s 已经禁用秒传检测，直接上传\n", utu.taskInfo.Id(), time.Now().Format("2006-01-02 15:04:06"))
|
||||
@ -450,7 +441,7 @@ StepUploadPrepareUpload:
|
||||
// existed, delete it
|
||||
var fileDeleteResult []*aliyunpan.FileBatchActionResult
|
||||
var err *apierror.ApiError
|
||||
fileDeleteResult, err = utu.PanClient.FileDelete([]*aliyunpan.FileBatchActionParam{{DriveId: efi.DriveId, FileId: efi.FileId}})
|
||||
fileDeleteResult, err = utu.PanClient.OpenapiPanClient().FileDelete([]*aliyunpan.FileBatchActionParam{{DriveId: efi.DriveId, FileId: efi.FileId}})
|
||||
if err != nil || len(fileDeleteResult) == 0 {
|
||||
result.Err = err
|
||||
result.ResultMessage = "无法删除文件，请稍后重试"
|
||||
@ -482,16 +473,7 @@ StepUploadPrepareUpload:
|
||||
ProofVersion: "v1",
|
||||
}
|
||||
|
||||
uploadOpEntity, apierr = utu.PanClient.CreateUploadFile(appCreateUploadFileParam)
|
||||
if apierr != nil && apierr.Code == apierror.ApiCodeDeviceSessionSignatureInvalid {
|
||||
_, e := utu.PanClient.CreateSession(nil)
|
||||
if e == nil {
|
||||
// retry
|
||||
uploadOpEntity, apierr = utu.PanClient.CreateUploadFile(appCreateUploadFileParam)
|
||||
} else {
|
||||
logger.Verboseln("CreateSession failed")
|
||||
}
|
||||
}
|
||||
uploadOpEntity, apierr = utu.PanClient.OpenapiPanClient().CreateUploadFile(appCreateUploadFileParam)
|
||||
if apierr != nil {
|
||||
result.Err = apierr
|
||||
result.ResultMessage = "创建上传任务失败：" + apierr.Error()
|
||||
|
||||
@ -569,7 +569,9 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
|
||||
// 创建分片上传器
|
||||
// 阿里云盘默认就是分片上传，每一个分片对应一个part_info
|
||||
// 但是不支持分片同时上传，必须单线程，并且按照顺序从1开始一个一个上传
|
||||
worker := panupload.NewPanUpload(f.panClient, f.syncItem.getPanFileFullPath(), f.syncItem.DriveId, f.syncItem.UploadEntity, f.syncItem.UseInternalUrl)
|
||||
// TODO: need fix
|
||||
//worker := panupload.NewPanUpload(f.panClient, f.syncItem.getPanFileFullPath(), f.syncItem.DriveId, f.syncItem.UploadEntity, f.syncItem.UseInternalUrl)
|
||||
worker := panupload.NewPanUpload(nil, f.syncItem.getPanFileFullPath(), f.syncItem.DriveId, f.syncItem.UploadEntity, f.syncItem.UseInternalUrl)
|
||||
|
||||
// 限速配置
|
||||
var rateLimit *speeds.RateLimit
|
||||
|
||||
Loading…
Reference in New Issue
Block a user