From 9962e63c8179f9b4adf4c21c76ee2cdb2d40913c Mon Sep 17 00:00:00 2001 From: tickstep Date: Wed, 20 Mar 2024 18:04:30 +0800 Subject: [PATCH] add sync policy for download&upload mode --- internal/command/sync.go | 27 +++++++--- internal/syncdrive/file_action_task_mgr.go | 61 +++++++++++++++++++--- internal/syncdrive/sync_task.go | 59 +++++++++++++-------- internal/syncdrive/sync_task_mgr.go | 3 ++ 4 files changed, 114 insertions(+), 36 deletions(-) diff --git a/internal/command/sync.go b/internal/command/sync.go index deb6980..f7a2132 100644 --- a/internal/command/sync.go +++ b/internal/command/sync.go @@ -72,6 +72,7 @@ func CmdSync() cli.Command { "localFolderPath": "D:/tickstep/Documents/设计文档", "panFolderPath": "/sync_drive/我的文档", "mode": "upload", + "policy"： "increment"， "driveName": "backup" } ] @@ -81,6 +82,7 @@ name - 任务名称 localFolderPath - 本地目录 panFolderPath - 网盘目录 mode - 模式，支持两种: upload(备份本地文件到云盘),download(备份云盘文件到本地) +policy - 备份策略, 支持两种: exclusive(排他备份文件，目标目录多余的文件会被删除),increment(增量备份文件，目标目录多余的文件不会被删除) driveName - 网盘名称，backup(备份盘)，resource(资源盘) 例子: @@ -160,6 +162,7 @@ driveName - 网盘名称，backup(备份盘)，resource(资源盘) localDir := c.String("ldir") panDir := c.String("pdir") mode := c.String("mode") + policy := c.String("policy") driveName := c.String("drive") if localDir != "" && panDir != "" { // make path absolute @@ -188,15 +191,22 @@ driveName - 网盘名称，backup(备份盘)，resource(资源盘) task = &syncdrive.SyncTask{} task.LocalFolderPath = path.Clean(strings.ReplaceAll(localDir, "\\", "/")) task.PanFolderPath = panDir - task.Mode = syncdrive.UploadOnly - if mode == string(syncdrive.UploadOnly) { - task.Mode = syncdrive.UploadOnly - } else if mode == string(syncdrive.DownloadOnly) { - task.Mode = syncdrive.DownloadOnly + task.Mode = syncdrive.Upload + if mode == string(syncdrive.Upload) { + task.Mode = syncdrive.Upload + } else if mode == string(syncdrive.Download) { + task.Mode = syncdrive.Download } else if mode == string(syncdrive.SyncTwoWay) { task.Mode = syncdrive.SyncTwoWay } else { - task.Mode = syncdrive.UploadOnly + task.Mode = syncdrive.Upload + } + if policy == string(syncdrive.SyncPolicyExclusive) { + task.Policy = syncdrive.SyncPolicyExclusive + } else if policy == string(syncdrive.SyncPolicyIncrement) { + task.Policy = syncdrive.SyncPolicyIncrement + } else { + task.Policy = syncdrive.SyncPolicyIncrement } task.Name = path.Base(task.LocalFolderPath) task.Id = utils.Md5Str(task.LocalFolderPath) @@ -239,6 +249,11 @@ driveName - 网盘名称，backup(备份盘)，resource(资源盘) Usage: "备份模式, 支持两种: upload(备份本地文件到云盘),download(备份云盘文件到本地)", Value: "upload", }, + cli.StringFlag{ + Name: "policy", + Usage: "备份策略, 支持两种: exclusive(排他备份文件，目标目录多余的文件会被删除),increment(增量备份文件，目标目录多余的文件不会被删除)", + Value: "increment", + }, cli.IntFlag{ Name: "dp", Usage: "download parallel, 下载并发数量，即可以同时并发下载多少个文件。0代表跟从配置文件设置（取值范围:1 ~ 10）", diff --git a/internal/syncdrive/file_action_task_mgr.go b/internal/syncdrive/file_action_task_mgr.go index 3424bec..037f580 100644 --- a/internal/syncdrive/file_action_task_mgr.go +++ b/internal/syncdrive/file_action_task_mgr.go @@ -4,6 +4,8 @@ import ( "context" "fmt" mapset "github.com/deckarep/golang-set" + "github.com/tickstep/aliyunpan-api/aliyunpan" + "github.com/tickstep/aliyunpan-api/aliyunpan/apierror" "github.com/tickstep/aliyunpan/internal/config" "github.com/tickstep/aliyunpan/internal/plugins" "github.com/tickstep/aliyunpan/internal/utils" @@ -171,7 +173,7 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF // download file from pan drive if panFilesNeedToDownload != nil { for _, file := range panFilesNeedToDownload { - if f.task.Mode == DownloadOnly { + if f.task.Mode == Download { syncItem := &SyncFileItem{ Action: SyncFileActionDownload, Status: SyncFileStatusCreate, @@ -195,6 +197,13 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF } f.addToSyncDb(fileActionTask) } + } else if f.task.Mode == Upload { + if f.task.Policy == SyncPolicyExclusive { + // 需要删除云盘多余的文件 + if f.deletePanFile(file) == nil { + PromptPrintln("成功删除云盘多余文件：" + file.Path) + } + } } } } @@ -202,7 +211,7 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF // upload file to pan drive if localFilesNeedToUpload != nil { for _, file := range localFilesNeedToUpload { - if f.task.Mode == UploadOnly { + if f.task.Mode == Upload { // check local file modified or not if file.IsFile() { if f.syncOption.LocalFileModifiedCheckIntervalSec > 0 { @@ -238,6 +247,13 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF } f.addToSyncDb(fileActionTask) } + } else if f.task.Mode == Download { + if f.task.Policy == SyncPolicyExclusive { + // 需要删除云盘多余的文件 + if f.deleteLocalFile(file) == nil { + PromptPrintln("成功删除本地多余文件：" + file.Path) + } + } } } } @@ -254,7 +270,7 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF // 本地文件和云盘文件SHA1不一样 // 不同模式同步策略不一样 - if f.task.Mode == UploadOnly { + if f.task.Mode == Upload { // 不再这里计算SHA1，待到上传的时候再计算 //if localFile.Sha1Hash == "" { @@ -298,7 +314,7 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF }, } f.addToSyncDb(uploadLocalFile) - } else if f.task.Mode == DownloadOnly { + } else if f.task.Mode == Download { // 校验SHA1是否相同 if strings.ToLower(panFile.Sha1Hash) == strings.ToLower(localFile.Sha1Hash) { // do nothing @@ -327,7 +343,7 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF } } -// 创建本地文件夹 +// createLocalFolder 创建本地文件夹 func (f *FileActionTaskManager) createLocalFolder(panFileItem *PanFileItem) error { panPath := panFileItem.Path panPath = strings.ReplaceAll(panPath, "\\", "/") @@ -368,6 +384,37 @@ func (f *FileActionTaskManager) createPanFolder(localFileItem *LocalFileItem) er } } +// deleteLocalFile 删除本地文件 +func (f *FileActionTaskManager) deleteLocalFile(localFileItem *LocalFileItem) error { + localFilePath := localFileItem.Path + logger.Verbosef("正在删除本地文件: %s\n", localFilePath) + var e error + if localFileItem.IsFolder() { + e = os.RemoveAll(localFilePath) + } else { + e = os.Remove(localFilePath) + } + if e == nil { + logger.Verbosef("删除本地文件成功: %s\n", localFilePath) + return nil + } + return e +} + +// deletePanFile 删除云盘文件 +func (f *FileActionTaskManager) deletePanFile(panFileItem *PanFileItem) error { + logger.Verbosef("正在删除云盘文件: %s\n", panFileItem.Path) + var fileDeleteResult *aliyunpan.FileBatchActionResult + var err *apierror.ApiError = nil + fileDeleteResult, err = f.task.panClient.OpenapiPanClient().FileDeleteCompletely(&aliyunpan.FileBatchActionParam{DriveId: panFileItem.DriveId, FileId: panFileItem.FileId}) + time.Sleep(1 * time.Second) + if err == nil && fileDeleteResult.Success { + logger.Verbosef("删除云盘文件成功: %s\n", panFileItem.Path) + return nil + } + return err +} + func (f *FileActionTaskManager) addToSyncDb(fileTask *FileActionTask) { f.mutex.Lock() defer f.mutex.Unlock() @@ -553,9 +600,9 @@ func (f *FileActionTaskManager) fileActionTaskExecutor(ctx context.Context) { f.setExecuteLoopFlag(true) logger.Verboseln("file execute task is finish, exit normally") prompt := "" - if f.task.Mode == UploadOnly { + if f.task.Mode == Upload { prompt = "完成全部文件的同步上传，等待下一次扫描" - } else if f.task.Mode == DownloadOnly { + } else if f.task.Mode == Download { prompt = "完成全部文件的同步下载，等待下一次扫描" } else { prompt = "完成全部文件的同步，等待下一次扫描" diff --git a/internal/syncdrive/sync_task.go b/internal/syncdrive/sync_task.go index 821eb5c..747fdeb 100644 --- a/internal/syncdrive/sync_task.go +++ b/internal/syncdrive/sync_task.go @@ -20,9 +20,9 @@ import ( ) type ( - TaskStep string - SyncMode string - CycleMode string + SyncMode string + SyncPolicy string + CycleMode string // SyncTask 同步任务 SyncTask struct { @@ -40,8 +40,10 @@ type ( LocalFolderPath string `json:"localFolderPath"` // PanFolderPath 云盘目录 PanFolderPath string `json:"panFolderPath"` - // Mode 同步模式 + // Mode 备份模式 Mode SyncMode `json:"mode"` + // Policy 备份策略 + Policy SyncPolicy `json:"policy"` // CycleMode 循环模式，OneTime-运行一次，InfiniteLoop-无限循环模式 CycleModeType CycleMode `json:"-"` // Priority 优先级选项 @@ -73,24 +75,22 @@ type ( ) const ( - // UploadOnly 单向上传，即备份本地文件到云盘 - UploadOnly SyncMode = "upload" - // DownloadOnly 只下载，即备份云盘文件到本地 - DownloadOnly SyncMode = "download" + // Upload 上传，即备份本地文件到云盘 + Upload SyncMode = "upload" + // Download 下载，即备份云盘文件到本地 + Download SyncMode = "download" // SyncTwoWay 双向同步，本地和云盘文件完全保持一致 SyncTwoWay SyncMode = "sync" + // SyncPolicyExclusive 备份策略，排他备份，保证本地和云盘一比一备份，目标目录多余的文件会被删除 + SyncPolicyExclusive SyncPolicy = "exclusive" + // SyncPolicyIncrement 备份策略，增量备份，只会增量备份文件，目标目录多余的（旧的）文件不会被删除 + SyncPolicyIncrement SyncPolicy = "increment" + // CycleOneTime 只运行一次 CycleOneTime CycleMode = "OneTime" // CycleInfiniteLoop 无限循环模式 CycleInfiniteLoop CycleMode = "InfiniteLoop" - - // StepScanFile 任务步骤，扫描文件建立同步数据库 - StepScanFile TaskStep = "scan" - // StepDiffFile 任务步骤，对比文件 - StepDiffFile TaskStep = "diff" - // StepSyncFile 任务步骤，同步文件 - StepSyncFile TaskStep = "sync" ) func (t *SyncTask) NameLabel() string { @@ -101,11 +101,11 @@ func (t *SyncTask) String() string { builder := &strings.Builder{} builder.WriteString("任务: " + t.NameLabel() + "\n") mode := "双向备份" - if t.Mode == UploadOnly { - mode = "备份本地文件（只上传）" + if t.Mode == Upload { + mode = "备份本地文件（上传）" } - if t.Mode == DownloadOnly { - mode = "备份云盘文件（只下载）" + if t.Mode == Download { + mode = "备份云盘文件（下载）" } builder.WriteString("同步模式: " + mode + "\n") if t.Mode == SyncTwoWay { @@ -119,6 +119,14 @@ func (t *SyncTask) String() string { } builder.WriteString("优先选项: " + priority + "\n") } + policy := "增量备份" + if t.Policy == SyncPolicyExclusive { + policy = "排他备份（上传&删除）" + } + if t.Policy == SyncPolicyIncrement { + policy = "增量备份（只上传）" + } + builder.WriteString("同步策略: " + policy + "\n") builder.WriteString("本地目录: " + t.LocalFolderPath + "\n") builder.WriteString("云盘目录: " + t.PanFolderPath + "\n") driveName := "备份盘" @@ -209,11 +217,16 @@ func (t *SyncTask) Start() error { return e } + // 策略 + if t.Policy == "" { + t.Policy = SyncPolicyIncrement + } + // 启动文件扫描进程 t.SetScanLoopFlag(false) - if t.Mode == UploadOnly { + if t.Mode == Upload { go t.scanLocalFile(t.ctx) - } else if t.Mode == DownloadOnly { + } else if t.Mode == Download { go t.scanPanFile(t.ctx) } else { return fmt.Errorf("异常：暂不支持该模式。") @@ -333,7 +346,7 @@ func (t *SyncTask) discardLocalFileDb(filePath string, startTimeUnix int64) bool } for _, file := range files { if file.ScanTimeAt == "" || file.ScanTimeUnix() < startTimeUnix { - if t.Mode == DownloadOnly { + if t.Mode == Download { // delete discard local file info directly t.localFileDb.Delete(file.Path) logger.Verboseln("label discard local file from DB: ", utils.ObjectToJsonStr(file, false)) @@ -572,7 +585,7 @@ func (t *SyncTask) discardPanFileDb(filePath string, startTimeUnix int64) bool { } for _, file := range files { if file.ScanTimeUnix() < startTimeUnix { - if t.Mode == UploadOnly { + if t.Mode == Upload { // delete discard pan file info directly t.panFileDb.Delete(file.Path) logger.Verboseln("delete discard pan file from DB: ", utils.ObjectToJsonStr(file, false)) diff --git a/internal/syncdrive/sync_task_mgr.go b/internal/syncdrive/sync_task_mgr.go index ef47106..4a297a2 100644 --- a/internal/syncdrive/sync_task_mgr.go +++ b/internal/syncdrive/sync_task_mgr.go @@ -180,6 +180,9 @@ func (m *SyncTaskManager) Start(tasks []*SyncTask) (bool, error) { task.Priority = SyncPriorityTimestampFirst task.syncOption.SyncPriority = SyncPriorityTimestampFirst } + if task.Policy == "" { + task.Policy = SyncPolicyIncrement + } task.LocalFolderPath = path.Clean(task.LocalFolderPath) task.PanFolderPath = path.Clean(task.PanFolderPath) if e := task.Start(); e != nil {