Clone/aliyunpan
1
0
Fork 0
mirror of https://github.com/tickstep/aliyunpan.git synced 2025-02-02 21:27:15 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

not support upload & download using InternalUrl

Browse Source
This commit is contained in:
tickstep 2024-03-03 22:22:50 +08:00
parent 6aecb98595
commit 443eac7589
16 changed files with 11 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/command/album.go
View File

@ -652,7 +652,6 @@ func RunAlbumDownloadFile(albumNames []string, options *DownloadOptions) {
MaxRate: config.Config.MaxDownloadRate,
InstanceStateStorageFormat: downloader.InstanceStateStorageFormatJSON,
ShowProgress: options.ShowProgress,
UseInternalUrl: config.Config.TransferUrlType == 2,
ExcludeNames: options.ExcludeNames,
}
if cfg.CacheSize == 0 {

18
internal/command/command.go
View File

@ -24,14 +24,7 @@ import (
"github.com/urfave/cli"
)
type (
// 秒传数据项
RapidUploadItem struct {
FileSha1 string
FileSize int64
FilePath string // 绝对路径，包含文件名
}
)
type ()
const (
cryptoDescription = `
@ -128,10 +121,6 @@ func CmdConfig() cli.Command {
return nil
}
}
if c.IsSet("transfer_url_type") {
config.Config.TransferUrlType = c.Int("transfer_url_type")
}
if c.IsSet("savedir") {
config.Config.SaveDir = c.String("savedir")
}
@ -183,11 +172,6 @@ func CmdConfig() cli.Command {
Name: "max_upload_rate",
Usage: "限制最大上传速度, 0代表不限制",
},
cli.IntFlag{
Name: "transfer_url_type",
Usage: "上传下载URL类别，1-默认，2-阿里云ECS",
Value: 1,
},
cli.StringFlag{
Name: "savedir",
Usage: "下载文件的储存目录",

2
internal/command/download.go
View File

@ -46,7 +46,6 @@ type (
NoCheck bool
ShowProgress bool
DriveId string
UseInternalUrl bool // 是否使用内置链接
ExcludeNames []string // 排除的文件名，包括文件夹和文件。即这些文件/文件夹不进行下载，支持正则表达式
}
@ -256,7 +255,6 @@ func RunDownload(paths []string, options *DownloadOptions) {
MaxRate: config.Config.MaxDownloadRate,
InstanceStateStorageFormat: downloader.InstanceStateStorageFormatJSON,
ShowProgress: options.ShowProgress,
UseInternalUrl: config.Config.TransferUrlType == 2,
ExcludeNames: options.ExcludeNames,
}
if cfg.CacheSize == 0 {

10
internal/command/sync.go
View File

@ -297,7 +297,6 @@ priority - 优先级，只对双向同步备份模式有效。选项支持三种
func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadParallel int, downloadBlockSize, uploadBlockSize int64,
flag syncdrive.SyncPriorityOption, localDelayTime int, taskStep syncdrive.TaskStep) {
useInternalUrl := config.Config.TransferUrlType == 2
maxDownloadRate := config.Config.MaxDownloadRate
maxUploadRate := config.Config.MaxUploadRate
activeUser := GetActiveUser()
@ -335,10 +334,6 @@ func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadPa
}
fmt.Println("启动同步备份进程")
typeUrlStr := "默认链接"
if useInternalUrl {
typeUrlStr = "阿里ECS内部链接"
}
// 文件同步记录器
fileRecorder := log.NewFileRecorder(config.GetLogDir() + "/sync_file_records.csv")
@ -348,7 +343,6 @@ func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadPa
FileUploadParallel: fileUploadParallel,
FileDownloadBlockSize: downloadBlockSize,
FileUploadBlockSize: uploadBlockSize,
UseInternalUrl: useInternalUrl,
MaxDownloadRate: maxDownloadRate,
MaxUploadRate: maxUploadRate,
SyncPriority: flag,
@ -360,8 +354,8 @@ func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadPa
if tasks != nil {
syncConfigFile = "(使用命令行配置)"
}
fmt.Printf("备份配置文件：%s\n链接类型：%s\n下载并发：%d\n上传并发：%d\n下载分片大小：%s\n上传分片大小：%s\n",
syncConfigFile, typeUrlStr, fileDownloadParallel, fileUploadParallel, converter.ConvertFileSize(downloadBlockSize, 2),
fmt.Printf("备份配置文件：%s\n下载并发：%d\n上传并发：%d\n下载分片大小：%s\n上传分片大小：%s\n",
syncConfigFile, fileDownloadParallel, fileUploadParallel, converter.ConvertFileSize(downloadBlockSize, 2),
converter.ConvertFileSize(uploadBlockSize, 2))
if _, e := syncMgr.Start(tasks, taskStep); e != nil {
fmt.Println("启动任务失败：", e)

3
internal/command/upload.go
View File

@ -65,7 +65,6 @@ type (
DriveId string
ExcludeNames []string // 排除的文件名，包括文件夹和文件。即这些文件/文件夹不进行上传，支持正则表达式
BlockSize int64 // 分片大小
UseInternalUrl bool // 是否使用内置链接
}
)
@ -259,7 +258,6 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
if opt.MaxRetry < 0 {
opt.MaxRetry = DefaultUploadMaxRetry
}
opt.UseInternalUrl = config.Config.TransferUrlType == 2
// 超时时间
if opt.MaxTimeoutSec > 0 {
@ -398,7 +396,6 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
ShowProgress: opt.ShowProgress,
IsOverwrite: opt.IsOverwrite,
IsSkipSameName: opt.IsSkipSameName,
UseInternalUrl: opt.UseInternalUrl,
GlobalSpeedsStat: globalSpeedsStat,
FileRecorder: fileRecorder,
}, opt.MaxRetry)

1
internal/config/pan_config.go
View File

@ -93,7 +93,6 @@ type PanConfig struct {
MaxDownloadRate int64 `json:"maxDownloadRate"` // 限制最大下载速度，单位 B/s, 即字节/每秒
MaxUploadRate int64 `json:"maxUploadRate"` // 限制最大上传速度，单位 B/s, 即字节/每秒
TransferUrlType int `json:"transferUrlType"` // 上传/下载URL类别，1-默认，2-阿里云ECS
SaveDir string `json:"saveDir"` // 下载储存路径

1
internal/config/pan_config_export.go
View File

@ -110,7 +110,6 @@ func (c *PanConfig) PrintTable() {
[]string{"max_upload_parallel", strconv.Itoa(c.MaxUploadParallel), "1 ~ 20", "最大上传并发量，即同时上传文件最大数量"},
[]string{"max_download_rate", showMaxRate(c.MaxDownloadRate), "", "限制单个文件最大下载速度, 0代表不限制"},
[]string{"max_upload_rate", showMaxRate(c.MaxUploadRate), "", "限制单个文件最大上传速度, 0代表不限制"},
[]string{"transfer_url_type", strconv.Itoa(c.TransferUrlType), "1-默认，2-阿里云ECS", "上传下载URL类别。除非在阿里云ECS（暂只支持经典网络）服务器中使用，不然请设置1"},
[]string{"savedir", c.SaveDir, "", "下载文件的储存目录"},
[]string{"proxy", c.Proxy, "", "设置代理, 支持 http/socks5 代理，例如: http://127.0.0.1:8888 或者 socks5://127.0.0.1:8889"},
[]string{"local_addrs", c.LocalAddrs, "", "设置本地网卡地址, 多个地址用逗号隔开，例如: 127.0.0.1,192.168.100.126"},

1
internal/file/downloader/config.go
View File

@ -41,7 +41,6 @@ type Config struct {
InstanceStatePath string // 断点续传信息路径
TryHTTP bool // 是否尝试使用 http 连接
ShowProgress bool // 是否展示下载进度条
UseInternalUrl bool // 是否使用内置链接
ExcludeNames []string // 排除的文件名，包括文件夹和文件。即这些文件/文件夹不进行下载，支持正则表达式
}

3
internal/file/downloader/downloader.go
View File

@ -411,9 +411,6 @@ func (der *Downloader) Execute() error {
client.SetTimeout(10 * time.Minute)
realUrl := durl.Url
if der.config.UseInternalUrl {
realUrl = durl.InternalUrl
}
worker := NewWorker(k, der.driveId, der.fileInfo.FileId, realUrl, writer, der.globalSpeedsStat)
worker.SetClient(client)
worker.SetPanClient(der.panClient)

13
internal/functions/panupload/upload.go
View File

@ -38,7 +38,6 @@ type (
// 网盘上传参数
uploadOpEntity *aliyunpan.CreateFileUploadResult
useInternalUrl bool
}
EmptyReaderLen64 struct {
@ -53,13 +52,12 @@ func (e EmptyReaderLen64) Len() int64 {
return 0
}
func NewPanUpload(panClient *config.PanClient, targetPath, driveId string, uploadOpEntity *aliyunpan.CreateFileUploadResult, useInternalUrl bool) uploader.MultiUpload {
func NewPanUpload(panClient *config.PanClient, targetPath, driveId string, uploadOpEntity *aliyunpan.CreateFileUploadResult) uploader.MultiUpload {
return &PanUpload{
panClient: panClient,
targetPath: targetPath,
driveId: driveId,
uploadOpEntity: uploadOpEntity,
useInternalUrl: useInternalUrl,
}
}
@ -78,9 +76,6 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
// check url expired or not
uploadUrl := pu.uploadOpEntity.PartInfoList[partseq].UploadURL
if pu.useInternalUrl {
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL
}
if IsUrlExpired(uploadUrl) {
// get renew upload url
infoList := make([]aliyunpan.FileUploadPartInfoParam, 0)
@ -183,9 +178,6 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
// 上传一个分片数据
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].UploadURL
if pu.useInternalUrl {
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL
}
apiError := pu.panClient.OpenapiPanClient().UploadFileData(uploadUrl, uploadFunc)
if respErr != nil {
@ -206,9 +198,6 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
// 获取新的上传URL重试一次
pu.uploadOpEntity.PartInfoList[partseq] = guur.PartInfoList[0]
uploadUrl := pu.uploadOpEntity.PartInfoList[partseq].UploadURL
if pu.useInternalUrl {
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL
}
apiError = pu.panClient.OpenapiPanClient().UploadFileData(uploadUrl, uploadFunc)
} else if respErr.Err == uploader.UploadPartAlreadyExist {
// already upload

5
internal/functions/panupload/upload_task_unit.go
View File

@ -73,9 +73,6 @@ type (
IsOverwrite bool // 覆盖已存在的文件，如果同名文件已存在则移到回收站里
IsSkipSameName bool // 跳过已存在的文件，即使文件内容不一致(不检查SHA1)
// 是否使用内置链接
UseInternalUrl bool
// 全局速度统计
GlobalSpeedsStat *speeds.Speeds
@ -165,7 +162,7 @@ func (utu *UploadTaskUnit) upload() (result *taskframework.TaskUnitRunResult) {
// 阿里云盘默认就是分片上传，每一个分片对应一个part_info
// 但是不支持分片同时上传，必须单线程，并且按照顺序从1开始一个一个上传
muer := uploader.NewMultiUploader(
NewPanUpload(utu.PanClient, utu.SavePath, utu.DriveId, utu.LocalFileChecksum.UploadOpEntity, utu.UseInternalUrl),
NewPanUpload(utu.PanClient, utu.SavePath, utu.DriveId, utu.LocalFileChecksum.UploadOpEntity),
rio.NewFileReaderAtLen64(utu.LocalFileChecksum.GetFile()), &uploader.MultiUploaderConfig{
Parallel: utu.Parallel,
BlockSize: utu.BlockSize,

8
internal/syncdrive/file_action_task.go
View File

@ -300,9 +300,6 @@ func (f *FileActionTask) downloadFile(ctx context.Context) error {
}
downloadUrl := durl.Url
if f.syncItem.UseInternalUrl {
downloadUrl = durl.InternalUrl
}
worker := downloader.NewWorker(0, f.syncItem.PanFile.DriveId, f.syncItem.PanFile.FileId, downloadUrl, writer, nil)
// 限速
@ -554,9 +551,6 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
// 检测链接是否过期
// check url expired or not
uploadUrl := f.syncItem.UploadEntity.PartInfoList[f.syncItem.UploadPartSeq].UploadURL
if f.syncItem.UseInternalUrl {
uploadUrl = f.syncItem.UploadEntity.PartInfoList[f.syncItem.UploadPartSeq].InternalUploadURL
}
if panupload.IsUrlExpired(uploadUrl) {
// get renew upload url
logger.Verbosef("链接过期，获取新的上传链接: %s\n", targetPanFilePath)
@ -584,7 +578,7 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
// 创建分片上传器
// 阿里云盘默认就是分片上传，每一个分片对应一个part_info
// 但是不支持分片同时上传，必须单线程，并且按照顺序从1开始一个一个上传
worker := panupload.NewPanUpload(f.panClient, f.syncItem.getPanFileFullPath(), f.syncItem.DriveId, f.syncItem.UploadEntity, f.syncItem.UseInternalUrl)
worker := panupload.NewPanUpload(f.panClient, f.syncItem.getPanFileFullPath(), f.syncItem.DriveId, f.syncItem.UploadEntity)
// 限速配置
var rateLimit *speeds.RateLimit

10
internal/syncdrive/file_action_task_mgr.go
View File

@ -347,7 +347,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
DriveId: f.task.DriveId,
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
UploadBlockSize: f.syncOption.FileUploadBlockSize,
UseInternalUrl: f.syncOption.UseInternalUrl,
}
if file.IsFolder() {
@ -379,7 +378,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
DriveId: f.task.DriveId,
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
UploadBlockSize: f.syncOption.FileUploadBlockSize,
UseInternalUrl: f.syncOption.UseInternalUrl,
},
}
f.addToSyncDb(fileActionTask)
@ -420,7 +418,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
DriveId: f.task.DriveId,
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
UploadBlockSize: f.syncOption.FileUploadBlockSize,
UseInternalUrl: f.syncOption.UseInternalUrl,
}
if file.IsFolder() {
if localFolderQueue != nil {
@ -451,7 +448,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
DriveId: f.task.DriveId,
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
UploadBlockSize: f.syncOption.FileUploadBlockSize,
UseInternalUrl: f.syncOption.UseInternalUrl,
},
}
f.addToSyncDb(fileActionTask)
@ -492,7 +488,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
DriveId: f.task.DriveId,
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
UploadBlockSize: f.syncOption.FileUploadBlockSize,
UseInternalUrl: f.syncOption.UseInternalUrl,
},
}
f.addToSyncDb(deletePanFile)
@ -517,7 +512,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
DriveId: f.task.DriveId,
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
UploadBlockSize: f.syncOption.FileUploadBlockSize,
UseInternalUrl: f.syncOption.UseInternalUrl,
},
}
f.addToSyncDb(deletePanFile)
@ -586,7 +580,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
DriveId: f.task.DriveId,
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
UploadBlockSize: f.syncOption.FileUploadBlockSize,
UseInternalUrl: f.syncOption.UseInternalUrl,
},
}
f.addToSyncDb(uploadLocalFile)
@ -603,7 +596,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
DriveId: f.task.DriveId,
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
UploadBlockSize: f.syncOption.FileUploadBlockSize,
UseInternalUrl: f.syncOption.UseInternalUrl,
},
}
f.addToSyncDb(downloadPanFile)
@ -644,7 +636,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
DriveId: f.task.DriveId,
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
UploadBlockSize: f.syncOption.FileUploadBlockSize,
UseInternalUrl: f.syncOption.UseInternalUrl,
},
}
f.addToSyncDb(uploadLocalFile)
@ -661,7 +652,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
DriveId: f.task.DriveId,
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
UploadBlockSize: f.syncOption.FileUploadBlockSize,
UseInternalUrl: f.syncOption.UseInternalUrl,
},
}
f.addToSyncDb(downloadPanFile)

78
internal/syncdrive/file_action_task_mgr_test.go
View File

@ -1,78 +0,0 @@
package syncdrive
import (
"fmt"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan/internal/utils"
"github.com/tickstep/library-go/logger"
"os"
"testing"
"time"
)
func TestFileActionMgrStart(t *testing.T) {
refreshToken := "1640cc2d4ea...cb4d6242161a7"
webToken, err := aliyunpan.GetAccessTokenFromRefreshToken(refreshToken)
if err != nil {
fmt.Println("get acccess token error")
return
}
// pan client
panClient := aliyunpan.NewPanClient(*webToken, aliyunpan.AppLoginToken{}, aliyunpan.AppConfig{
AppId: "25dzX3vbYqktVxyX",
DeviceId: "E75459EXhOTkI5ZI6S3qDHA3",
UserId: "",
Nonce: 0,
PublicKey: "",
}, aliyunpan.SessionConfig{
DeviceName: "Chrome浏览器",
ModelName: "Windows网页版",
})
user, _ := panClient.GetUserInfo()
task := &SyncTask{
Id: "5b2d7c10-e927-4e72-8f9d-5abb3bb04814",
DriveId: user.FileDriveId,
LocalFolderPath: "D:\\smb\\feny\\goprojects\\dev\\NS游戏备份",
PanFolderPath: "/sync_drive",
Mode: "sync",
LastSyncTime: "2022-05-26 19:29:19",
syncDbFolderPath: "D:\\smb\\feny\\goprojects\\dev\\sync_drive",
panClient: panClient,
syncOption: SyncOption{
FileDownloadParallel: 2,
FileUploadParallel: 2,
FileDownloadBlockSize: int64(256 * 1024),
FileUploadBlockSize: aliyunpan.DefaultChunkSize,
UseInternalUrl: false,
MaxDownloadRate: 0,
MaxUploadRate: 0,
},
}
task.setupDb()
ft := NewFileActionTaskManager(task)
ft.Start()
//go func() {
// time.Sleep(10 * time.Second)
// task.Stop()
//}()
time.Sleep(50 * time.Minute)
ft.Stop()
}
func TestFileTime(t *testing.T) {
ts := utils.ParseTimeStr("2013-01-11 12:15:07")
//ts = time.Now()
if err := os.Chtimes("/Volumes/Downloads/dev/upload/password.key", ts, ts); err != nil {
logger.Verbosef(err.Error())
}
}
func TestLocalFileTime(t *testing.T) {
if file, er := os.Stat("/Volumes/Downloads/dev/upload/password.key"); er == nil {
fmt.Println(file.ModTime().Format("2006-01-02 15:04:05"))
}
}

1
internal/syncdrive/sync_db.go
View File

@ -133,7 +133,6 @@ type (
StatusUpdateTime string `json:"statusUpdateTime"`
DriveId string `json:"driveId"`
UseInternalUrl bool `json:"useInternalUrl"`
DownloadRange *transfer.Range `json:"downloadRange"`
DownloadBlockSize int64 `json:"downloadBlockSize"`
UploadRange *transfer.Range `json:"uploadRange"`

1
internal/syncdrive/sync_task_mgr.go
View File

@ -19,7 +19,6 @@ type (
FileUploadParallel int // 文件上传并发数
FileDownloadBlockSize int64 // 文件下载分片大小
FileUploadBlockSize int64 // 文件上传分片大小
UseInternalUrl bool // 是否使用阿里ECS内部链接
MaxDownloadRate int64 // 限制最大下载速度
MaxUploadRate int64 // 限制最大上传速度