From f692e2d18b34512e3778d9af0c8a2ea8082cd8b2 Mon Sep 17 00:00:00 2001 From: xiaoyaofenfen <1254525673@qq.com> Date: Thu, 9 Jun 2022 19:58:16 +0800 Subject: [PATCH] add sync command --- internal/command/sync.go | 102 +++++++++++++++++++++ internal/config/pan_config.go | 6 ++ internal/syncdrive/file_action_task_mgr.go | 36 ++++++-- internal/syncdrive/sync_task.go | 10 +- internal/syncdrive/sync_task_mgr.go | 10 +- main.go | 3 + 6 files changed, 147 insertions(+), 20 deletions(-) create mode 100644 internal/command/sync.go diff --git a/internal/command/sync.go b/internal/command/sync.go new file mode 100644 index 0000000..d20670b --- /dev/null +++ b/internal/command/sync.go @@ -0,0 +1,102 @@ +// Copyright (c) 2020 tickstep. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package command + +import ( + "fmt" + "github.com/tickstep/aliyunpan/cmder" + "github.com/tickstep/aliyunpan/internal/config" + "github.com/tickstep/aliyunpan/internal/syncdrive" + "github.com/tickstep/aliyunpan/internal/utils" + "github.com/tickstep/library-go/logger" + "github.com/urfave/cli" + "os" + "strings" + "time" +) + +func CmdSync() cli.Command { + return cli.Command{ + Name: "sync", + Usage: "同步备份功能(Beta)", + UsageText: cmder.App().Name + " sync", + Description: ` + 备份功能。指定本地目录和对应的一个网盘目录，以备份文件。 + 备份功能支持一下三种模式： + 1. 备份本地文件，即上传本地文件到网盘，始终保持本地文件有一个完整的备份在网盘 + 2. 备份云盘文件，即下载网盘文件到本地，始终保持网盘的文件有一个完整的备份在本地 + 3. 双向备份，保持网盘文件和本地文件严格一致 + +`, + Category: "阿里云盘", + Before: cmder.ReloadConfigFunc, + Action: func(c *cli.Context) error { + if config.Config.ActiveUser() == nil { + fmt.Println("未登录账号") + return nil + } + blockSize := int64(c.Int("bs") * 1024) + RunSync(blockSize) + return nil + }, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "log", + Usage: "开启log输出", + }, + cli.IntFlag{ + Name: "bs", + Usage: "block size，上传分片大小，单位KB。推荐值：1024 ~ 10240", + Value: 10240, + }, + }, + } +} + +func RunSync(uploadBlockSize int64) { + activeUser := GetActiveUser() + panClient := activeUser.PanClient() + + // pan token expired checker + go func() { + for { + time.Sleep(time.Duration(1) * time.Minute) + if RefreshTokenInNeed(activeUser) { + logger.Verboseln("update access token for sync task") + panClient.UpdateToken(activeUser.WebToken) + } + } + }() + + syncFolderRootPath := config.GetSyncDriveDir() + if b, e := utils.PathExists(syncFolderRootPath); e == nil { + if !b { + os.MkdirAll(syncFolderRootPath, 0600) + } + } + + fmt.Println("启动同步备份进程") + syncMgr := syncdrive.NewSyncTaskManager(activeUser.DriveList.GetFileDriveId(), panClient, syncFolderRootPath) + if _, e := syncMgr.Start(); e != nil { + fmt.Println("启动任务失败：", e) + return + } + c := "" + for strings.ToLower(c) != "y" { + fmt.Print("本命令不会退出，如需要结束同步备份进程请输入y，然后按Enter键进行停止：") + fmt.Scan(&c) + } + fmt.Println("正在停止同步备份任务，请稍等...") + syncMgr.Stop() +} diff --git a/internal/config/pan_config.go b/internal/config/pan_config.go index 9676323..fd02e3d 100644 --- a/internal/config/pan_config.go +++ b/internal/config/pan_config.go @@ -308,10 +308,16 @@ func GetConfigDir() string { return configDir } +// GetPluginDir 获取插件文件夹路径 func GetPluginDir() string { return strings.TrimSuffix(GetConfigDir(), "/") + "/plugin" } +// GetSyncDriveDir 获取同步备份的文件夹路径 +func GetSyncDriveDir() string { + return strings.TrimSuffix(GetConfigDir(), "/") + "/sync_drive" +} + func (c *PanConfig) ActiveUser() *PanUser { if c.activeUser == nil { if c.UserList == nil { diff --git a/internal/syncdrive/file_action_task_mgr.go b/internal/syncdrive/file_action_task_mgr.go index cace8e5..e4288d3 100644 --- a/internal/syncdrive/file_action_task_mgr.go +++ b/internal/syncdrive/file_action_task_mgr.go @@ -235,10 +235,10 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil localFilesNeedToCheck, panFilesNeedToCheck := localFilesSet.Intersection(panFilesSet) // download file from pan drive - if f.task.Mode == DownloadOnly || f.task.Mode == SyncTwoWay { - if panFilesNeedToDownload != nil { - for _, file := range panFilesNeedToDownload { - if file.ScanStatus == ScanStatusNormal { // 下载文件 + if panFilesNeedToDownload != nil { + for _, file := range panFilesNeedToDownload { + if file.ScanStatus == ScanStatusNormal { // 下载文件 + if f.task.Mode == DownloadOnly || f.task.Mode == SyncTwoWay { if file.IsFolder() { if panFolderQueue != nil { panFolderQueue.PushUnique(file) @@ -259,7 +259,9 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil }, } f.addToSyncDb(fileActionTask) - } else if file.ScanStatus == ScanStatusDiscard { // 删除对应本地文件（文件夹） + } + } else if file.ScanStatus == ScanStatusDiscard { // 删除对应本地文件（文件夹） + if f.task.Mode == DownloadOnly || f.task.Mode == SyncTwoWay { fileActionTask := &FileActionTask{ syncItem: &SyncFileItem{ Action: SyncFileActionDeleteLocal, @@ -274,16 +276,19 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil }, } f.addToSyncDb(fileActionTask) + } else if f.task.Mode == UploadOnly { + // 删除无用记录 + f.task.panFileDb.Delete(file.Path) } } } } // upload file to pan drive - if f.task.Mode == UploadOnly || f.task.Mode == SyncTwoWay { - if localFilesNeedToUpload != nil { - for _, file := range localFilesNeedToUpload { - if file.ScanStatus == ScanStatusNormal { // 上传文件到云盘 + if localFilesNeedToUpload != nil { + for _, file := range localFilesNeedToUpload { + if file.ScanStatus == ScanStatusNormal { // 上传文件到云盘 + if f.task.Mode == UploadOnly || f.task.Mode == SyncTwoWay { if file.IsFolder() { if localFolderQueue != nil { localFolderQueue.PushUnique(file) @@ -304,7 +309,9 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil }, } f.addToSyncDb(fileActionTask) - } else if file.ScanStatus == ScanStatusDiscard { // 删除对应云盘文件（文件夹） + } + } else if file.ScanStatus == ScanStatusDiscard { // 删除对应云盘文件（文件夹） + if f.task.Mode == UploadOnly || f.task.Mode == SyncTwoWay { fileActionTask := &FileActionTask{ syncItem: &SyncFileItem{ Action: SyncFileActionDeletePan, @@ -319,6 +326,9 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil }, } f.addToSyncDb(fileActionTask) + } else if f.task.Mode == DownloadOnly { + // 删除无用记录 + f.task.localFileDb.Delete(file.Path) } } } @@ -355,6 +365,9 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil }, } f.addToSyncDb(deletePanFile) + } else if f.task.Mode == DownloadOnly { + // 删除无用记录 + f.task.localFileDb.Delete(localFile.Path) } continue } @@ -375,6 +388,9 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil }, } f.addToSyncDb(deletePanFile) + } else if f.task.Mode == UploadOnly { + // 删除无用记录 + f.task.panFileDb.Delete(panFile.Path) } continue } diff --git a/internal/syncdrive/sync_task.go b/internal/syncdrive/sync_task.go index 5cddaef..e04c9bd 100644 --- a/internal/syncdrive/sync_task.go +++ b/internal/syncdrive/sync_task.go @@ -25,7 +25,7 @@ type ( // Id 任务ID Id string `json:"id"` // DriveId 网盘ID，目前支持文件网盘 - DriveId string `json:"driveId"` + DriveId string `json:"-"` // LocalFolderPath 本地目录 LocalFolderPath string `json:"localFolderPath"` // PanFolderPath 云盘目录 @@ -66,12 +66,12 @@ func (t *SyncTask) NameLabel() string { func (t *SyncTask) String() string { builder := &strings.Builder{} builder.WriteString("任务: " + t.NameLabel() + "\n") - mode := "双向同步" + mode := "双向备份" if t.Mode == UploadOnly { - mode = "只上传" + mode = "备份本地文件（只上传）" } - if t.Mode == UploadOnly { - mode = "只下载" + if t.Mode == DownloadOnly { + mode = "备份云盘文件（只下载）" } builder.WriteString("同步模式: " + mode + "\n") builder.WriteString("本地目录: " + t.LocalFolderPath + "\n") diff --git a/internal/syncdrive/sync_task_mgr.go b/internal/syncdrive/sync_task_mgr.go index 2ca32e6..ebfaadc 100644 --- a/internal/syncdrive/sync_task_mgr.go +++ b/internal/syncdrive/sync_task_mgr.go @@ -8,6 +8,7 @@ import ( "github.com/tickstep/library-go/logger" "io/ioutil" "path" + "time" ) type ( @@ -63,9 +64,9 @@ func (m *SyncTaskManager) parseConfigFile() error { m.syncDriveConfig = r if b, _ := utils.PathExists(configFilePath); b != true { - text := utils.ObjectToJsonStr(r, true) - ioutil.WriteFile(configFilePath, []byte(text), 0600) - return nil + //text := utils.ObjectToJsonStr(r, true) + //ioutil.WriteFile(configFilePath, []byte(text), 0600) + return fmt.Errorf("备份配置文件不存在") } data, e := ioutil.ReadFile(configFilePath) if e != nil { @@ -109,11 +110,10 @@ func (m *SyncTaskManager) Start() (bool, error) { } fmt.Println("\n启动同步任务") fmt.Println(task) + time.Sleep(200 * time.Millisecond) } // save config file ioutil.WriteFile(m.configFilePath(), []byte(utils.ObjectToJsonStr(m.syncDriveConfig, true)), 0600) - - // TODO: refresh panClient token to keep access token alive return true, nil } diff --git a/main.go b/main.go index 3d49f6b..9c155a0 100644 --- a/main.go +++ b/main.go @@ -435,6 +435,9 @@ func main() { // 备份 backup command.CmdBackup(), + // 同步备份 sync + command.CmdSync(), + // 上传文件/目录 upload command.CmdUpload(),