From a28ed463e77eed0be9b71167a60a427906920809 Mon Sep 17 00:00:00 2001 From: tickstep Date: Thu, 11 Nov 2021 23:24:56 +0800 Subject: [PATCH] support use the internal download upload url for aliyun ECS --- internal/command/command.go | 9 +++++++++ internal/command/download.go | 2 ++ internal/command/upload.go | 4 +++- internal/config/pan_config.go | 1 + internal/config/pan_config_export.go | 1 + internal/file/downloader/config.go | 1 + internal/file/downloader/downloader.go | 6 +++++- internal/functions/panupload/upload.go | 16 +++++++++++++--- internal/functions/panupload/upload_task_unit.go | 5 ++++- 9 files changed, 39 insertions(+), 6 deletions(-) diff --git a/internal/command/command.go b/internal/command/command.go index 6cc3206..e6dcb2a 100644 --- a/internal/command/command.go +++ b/internal/command/command.go @@ -217,6 +217,10 @@ func CmdConfig() cli.Command { return nil } } + if c.IsSet("transfer_url_type") { + config.Config.TransferUrlType = c.Int("transfer_url_type") + } + if c.IsSet("savedir") { config.Config.SaveDir = c.String("savedir") } @@ -263,6 +267,11 @@ func CmdConfig() cli.Command { Name: "max_upload_rate", Usage: "限制最大上传速度, 0代表不限制", }, + cli.IntFlag{ + Name: "transfer_url_type", + Usage: "上传下载URL类别，1-默认，2-阿里云ECS", + Value: 1, + }, cli.StringFlag{ Name: "savedir", Usage: "下载文件的储存目录", diff --git a/internal/command/download.go b/internal/command/download.go index 20a67ff..7cab91f 100644 --- a/internal/command/download.go +++ b/internal/command/download.go @@ -44,6 +44,7 @@ type ( NoCheck bool ShowProgress bool DriveId string + UseInternalUrl bool // 是否使用内置链接 } // LocateDownloadOption 获取下载链接可选参数 @@ -208,6 +209,7 @@ func RunDownload(paths []string, options *DownloadOptions) { MaxRate: config.Config.MaxDownloadRate, InstanceStateStorageFormat: downloader.InstanceStateStorageFormatJSON, ShowProgress: options.ShowProgress, + UseInternalUrl: config.Config.TransferUrlType == 2, } if cfg.CacheSize == 0 { cfg.CacheSize = int(DownloadCacheSize) diff --git a/internal/command/upload.go b/internal/command/upload.go index eb6ac13..99ca1e2 100644 --- a/internal/command/upload.go +++ b/internal/command/upload.go @@ -62,6 +62,7 @@ type ( DriveId string ExcludeNames []string // 排除的文件名，包括文件夹和文件。即这些文件/文件夹不进行上传，支持正则表达式 BlockSize int64 // 分片大小 + UseInternalUrl bool // 是否使用内置链接 } ) @@ -259,10 +260,10 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) { if opt.Parallel <= 0 { opt.Parallel = 1 } - if opt.MaxRetry < 0 { opt.MaxRetry = DefaultUploadMaxRetry } + opt.UseInternalUrl = config.Config.TransferUrlType == 2 savePath = activeUser.PathJoin(opt.DriveId, savePath) _, err1 := activeUser.PanClient().FileInfoByPath(opt.DriveId, savePath) @@ -409,6 +410,7 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) { ShowProgress: opt.ShowProgress, IsOverwrite: opt.IsOverwrite, FolderSyncDb: db, + UseInternalUrl: opt.UseInternalUrl, }, opt.MaxRetry) fmt.Printf("%s [%s] 加入上传队列: %s\n", time.Now().Format("2006-01-02 15:04:05"), taskinfo.Id(), file) diff --git a/internal/config/pan_config.go b/internal/config/pan_config.go index cdad5e3..0d0181e 100644 --- a/internal/config/pan_config.go +++ b/internal/config/pan_config.go @@ -69,6 +69,7 @@ type PanConfig struct { MaxDownloadRate int64 `json:"maxDownloadRate"` // 限制最大下载速度，单位 B/s, 即字节/每秒 MaxUploadRate int64 `json:"maxUploadRate"` // 限制最大上传速度，单位 B/s, 即字节/每秒 + TransferUrlType int `json:"transferUrlType"` // 上传/下载URL类别，1-默认，2-阿里云ECS SaveDir string `json:"saveDir"` // 下载储存路径 diff --git a/internal/config/pan_config_export.go b/internal/config/pan_config_export.go index 221afe8..14a16be 100644 --- a/internal/config/pan_config_export.go +++ b/internal/config/pan_config_export.go @@ -79,6 +79,7 @@ func (c *PanConfig) PrintTable() { []string{"max_download_load", strconv.Itoa(c.MaxDownloadLoad), "1 ~ 5", "同时进行下载文件的最大数量"}, []string{"max_download_rate", showMaxRate(c.MaxDownloadRate), "", "限制最大下载速度, 0代表不限制"}, []string{"max_upload_rate", showMaxRate(c.MaxUploadRate), "", "限制最大上传速度, 0代表不限制"}, + []string{"transfer_url_type", strconv.Itoa(c.TransferUrlType), "1-默认，2-阿里云ECS", "上传下载URL类别。除非在阿里云ECS服务器中使用，不然请设置1"}, []string{"savedir", c.SaveDir, "", "下载文件的储存目录"}, []string{"proxy", c.Proxy, "", "设置代理, 支持 http/socks5 代理，例如：http://127.0.0.1:8888"}, []string{"local_addrs", c.LocalAddrs, "", "设置本地网卡地址, 多个地址用逗号隔开"}, diff --git a/internal/file/downloader/config.go b/internal/file/downloader/config.go index 9405170..d51fee8 100644 --- a/internal/file/downloader/config.go +++ b/internal/file/downloader/config.go @@ -38,6 +38,7 @@ type Config struct { InstanceStatePath string // 断点续传信息路径 TryHTTP bool // 是否尝试使用 http 连接 ShowProgress bool // 是否展示下载进度条 + UseInternalUrl bool // 是否使用内置链接 } //NewConfig 返回默认配置 diff --git a/internal/file/downloader/downloader.go b/internal/file/downloader/downloader.go index 5b76c95..36470ce 100644 --- a/internal/file/downloader/downloader.go +++ b/internal/file/downloader/downloader.go @@ -389,7 +389,11 @@ func (der *Downloader) Execute() error { client.SetKeepAlive(true) client.SetTimeout(10 * time.Minute) - worker := NewWorker(k, der.driveId, der.fileInfo.FileId, durl.Url, writer) + realUrl := durl.Url + if der.config.UseInternalUrl { + realUrl = durl.InternalUrl + } + worker := NewWorker(k, der.driveId, der.fileInfo.FileId, realUrl, writer) worker.SetClient(client) worker.SetPanClient(der.panClient) worker.SetWriteMutex(writeMu) diff --git a/internal/functions/panupload/upload.go b/internal/functions/panupload/upload.go index 2791f6d..3480271 100644 --- a/internal/functions/panupload/upload.go +++ b/internal/functions/panupload/upload.go @@ -37,6 +37,7 @@ type ( // 网盘上传参数 uploadOpEntity *aliyunpan.CreateFileUploadResult + useInternalUrl bool } UploadedFileMeta struct { @@ -69,12 +70,13 @@ func (e EmptyReaderLen64) Len() int64 { return 0 } -func NewPanUpload(panClient *aliyunpan.PanClient, targetPath, driveId string, uploadOpEntity *aliyunpan.CreateFileUploadResult) uploader.MultiUpload { +func NewPanUpload(panClient *aliyunpan.PanClient, targetPath, driveId string, uploadOpEntity *aliyunpan.CreateFileUploadResult, useInternalUrl bool) uploader.MultiUpload { return &PanUpload{ panClient: panClient, targetPath: targetPath, driveId: driveId, uploadOpEntity: uploadOpEntity, + useInternalUrl: useInternalUrl, } } @@ -154,7 +156,11 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int } // 上传一个分片数据 - apiError := pu.panClient.UploadFileData(pu.uploadOpEntity.PartInfoList[partseq].UploadURL, uploadFunc) + uploadUrl := pu.uploadOpEntity.PartInfoList[partseq].UploadURL + if pu.useInternalUrl { + uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL + } + apiError := pu.panClient.UploadFileData(uploadUrl, uploadFunc) if respErr != nil { if respErr.Err == uploadUrlExpired { @@ -173,7 +179,11 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int // 获取新的上传URL重试一次 pu.uploadOpEntity.PartInfoList[partseq] = guur.PartInfoList[0] - apiError = pu.panClient.UploadFileData(pu.uploadOpEntity.PartInfoList[partseq].UploadURL, uploadFunc) + uploadUrl := pu.uploadOpEntity.PartInfoList[partseq].UploadURL + if pu.useInternalUrl { + uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL + } + apiError = pu.panClient.UploadFileData(uploadUrl, uploadFunc) } else if respErr.Err == uploadPartAlreadyExist { // already upload // success diff --git a/internal/functions/panupload/upload_task_unit.go b/internal/functions/panupload/upload_task_unit.go index 17eac18..faa9537 100644 --- a/internal/functions/panupload/upload_task_unit.go +++ b/internal/functions/panupload/upload_task_unit.go @@ -64,6 +64,9 @@ type ( ShowProgress bool IsOverwrite bool // 覆盖已存在的文件，如果同名文件已存在则移到回收站里 + + // 是否使用内置链接 + UseInternalUrl bool } ) @@ -147,7 +150,7 @@ func (utu *UploadTaskUnit) upload() (result *taskframework.TaskUnitRunResult) { // 阿里云盘默认就是分片上传，每一个分片对应一个part_info // 但是不支持分片同时上传，必须单线程，并且按照顺序从1开始一个一个上传 muer := uploader.NewMultiUploader( - NewPanUpload(utu.PanClient, utu.SavePath, utu.DriveId, utu.LocalFileChecksum.UploadOpEntity), + NewPanUpload(utu.PanClient, utu.SavePath, utu.DriveId, utu.LocalFileChecksum.UploadOpEntity, utu.UseInternalUrl), rio.NewFileReaderAtLen64(utu.LocalFileChecksum.GetFile()), &uploader.MultiUploaderConfig{ Parallel: utu.Parallel, BlockSize: utu.BlockSize,