Clone/aliyunpan
1
0
Fork 0
mirror of https://github.com/tickstep/aliyunpan.git synced 2025-01-23 14:32:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add sync policy for download&upload mode

Browse Source
This commit is contained in:
tickstep 2024-03-20 18:04:30 +08:00
parent f7b8d26964
commit 9962e63c81
4 changed files with 114 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
internal/command/sync.go
View File

@ -72,6 +72,7 @@ func CmdSync() cli.Command {
"localFolderPath": "D:/tickstep/Documents/设计文档",
"panFolderPath": "/sync_drive/我的文档",
"mode": "upload",
"policy"： "increment"，
"driveName": "backup"
}
]
@ -81,6 +82,7 @@ name - 任务名称
localFolderPath - 本地目录
panFolderPath - 网盘目录
mode - 模式，支持两种: upload(备份本地文件到云盘),download(备份云盘文件到本地)
policy - 备份策略, 支持两种: exclusive(排他备份文件，目标目录多余的文件会被删除),increment(增量备份文件，目标目录多余的文件不会被删除)
driveName - 网盘名称，backup(备份盘)，resource(资源盘)
例子:
@ -160,6 +162,7 @@ driveName - 网盘名称，backup(备份盘)，resource(资源盘)
localDir := c.String("ldir")
panDir := c.String("pdir")
mode := c.String("mode")
policy := c.String("policy")
driveName := c.String("drive")
if localDir != "" && panDir != "" {
// make path absolute
@ -188,15 +191,22 @@ driveName - 网盘名称，backup(备份盘)，resource(资源盘)
task = &syncdrive.SyncTask{}
task.LocalFolderPath = path.Clean(strings.ReplaceAll(localDir, "\\", "/"))
task.PanFolderPath = panDir
task.Mode = syncdrive.UploadOnly
if mode == string(syncdrive.UploadOnly) {
task.Mode = syncdrive.UploadOnly
} else if mode == string(syncdrive.DownloadOnly) {
task.Mode = syncdrive.DownloadOnly
task.Mode = syncdrive.Upload
if mode == string(syncdrive.Upload) {
task.Mode = syncdrive.Upload
} else if mode == string(syncdrive.Download) {
task.Mode = syncdrive.Download
} else if mode == string(syncdrive.SyncTwoWay) {
task.Mode = syncdrive.SyncTwoWay
} else {
task.Mode = syncdrive.UploadOnly
task.Mode = syncdrive.Upload
}
if policy == string(syncdrive.SyncPolicyExclusive) {
task.Policy = syncdrive.SyncPolicyExclusive
} else if policy == string(syncdrive.SyncPolicyIncrement) {
task.Policy = syncdrive.SyncPolicyIncrement
} else {
task.Policy = syncdrive.SyncPolicyIncrement
}
task.Name = path.Base(task.LocalFolderPath)
task.Id = utils.Md5Str(task.LocalFolderPath)
@ -239,6 +249,11 @@ driveName - 网盘名称，backup(备份盘)，resource(资源盘)
Usage: "备份模式, 支持两种: upload(备份本地文件到云盘),download(备份云盘文件到本地)",
Value: "upload",
},
cli.StringFlag{
Name: "policy",
Usage: "备份策略, 支持两种: exclusive(排他备份文件，目标目录多余的文件会被删除),increment(增量备份文件，目标目录多余的文件不会被删除)",
Value: "increment",
},
cli.IntFlag{
Name: "dp",
Usage: "download parallel, 下载并发数量，即可以同时并发下载多少个文件。0代表跟从配置文件设置（取值范围:1 ~ 10）",

61
internal/syncdrive/file_action_task_mgr.go
View File

@ -4,6 +4,8 @@ import (
"context"
"fmt"
mapset "github.com/deckarep/golang-set"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/aliyunpan/internal/plugins"
"github.com/tickstep/aliyunpan/internal/utils"
@ -171,7 +173,7 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF
// download file from pan drive
if panFilesNeedToDownload != nil {
for _, file := range panFilesNeedToDownload {
if f.task.Mode == DownloadOnly {
if f.task.Mode == Download {
syncItem := &SyncFileItem{
Action: SyncFileActionDownload,
Status: SyncFileStatusCreate,
@ -195,6 +197,13 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF
}
f.addToSyncDb(fileActionTask)
}
} else if f.task.Mode == Upload {
if f.task.Policy == SyncPolicyExclusive {
// 需要删除云盘多余的文件
if f.deletePanFile(file) == nil {
PromptPrintln("成功删除云盘多余文件：" + file.Path)
}
}
}
}
}
@ -202,7 +211,7 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF
// upload file to pan drive
if localFilesNeedToUpload != nil {
for _, file := range localFilesNeedToUpload {
if f.task.Mode == UploadOnly {
if f.task.Mode == Upload {
// check local file modified or not
if file.IsFile() {
if f.syncOption.LocalFileModifiedCheckIntervalSec > 0 {
@ -238,6 +247,13 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF
}
f.addToSyncDb(fileActionTask)
}
} else if f.task.Mode == Download {
if f.task.Policy == SyncPolicyExclusive {
// 需要删除云盘多余的文件
if f.deleteLocalFile(file) == nil {
PromptPrintln("成功删除本地多余文件：" + file.Path)
}
}
}
}
}
@ -254,7 +270,7 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF
// 本地文件和云盘文件SHA1不一样
// 不同模式同步策略不一样
if f.task.Mode == UploadOnly {
if f.task.Mode == Upload {
// 不再这里计算SHA1，待到上传的时候再计算
//if localFile.Sha1Hash == "" {
@ -298,7 +314,7 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF
},
}
f.addToSyncDb(uploadLocalFile)
} else if f.task.Mode == DownloadOnly {
} else if f.task.Mode == Download {
// 校验SHA1是否相同
if strings.ToLower(panFile.Sha1Hash) == strings.ToLower(localFile.Sha1Hash) {
// do nothing
@ -327,7 +343,7 @@ func (f *FileActionTaskManager) doFileDiffRoutine(localFiles LocalFileList, panF
}
}
// 创建本地文件夹
// createLocalFolder 创建本地文件夹
func (f *FileActionTaskManager) createLocalFolder(panFileItem *PanFileItem) error {
panPath := panFileItem.Path
panPath = strings.ReplaceAll(panPath, "\\", "/")
@ -368,6 +384,37 @@ func (f *FileActionTaskManager) createPanFolder(localFileItem *LocalFileItem) er
}
}
// deleteLocalFile 删除本地文件
func (f *FileActionTaskManager) deleteLocalFile(localFileItem *LocalFileItem) error {
localFilePath := localFileItem.Path
logger.Verbosef("正在删除本地文件: %s\n", localFilePath)
var e error
if localFileItem.IsFolder() {
e = os.RemoveAll(localFilePath)
} else {
e = os.Remove(localFilePath)
}
if e == nil {
logger.Verbosef("删除本地文件成功: %s\n", localFilePath)
return nil
}
return e
}
// deletePanFile 删除云盘文件
func (f *FileActionTaskManager) deletePanFile(panFileItem *PanFileItem) error {
logger.Verbosef("正在删除云盘文件: %s\n", panFileItem.Path)
var fileDeleteResult *aliyunpan.FileBatchActionResult
var err *apierror.ApiError = nil
fileDeleteResult, err = f.task.panClient.OpenapiPanClient().FileDeleteCompletely(&aliyunpan.FileBatchActionParam{DriveId: panFileItem.DriveId, FileId: panFileItem.FileId})
time.Sleep(1 * time.Second)
if err == nil && fileDeleteResult.Success {
logger.Verbosef("删除云盘文件成功: %s\n", panFileItem.Path)
return nil
}
return err
}
func (f *FileActionTaskManager) addToSyncDb(fileTask *FileActionTask) {
f.mutex.Lock()
defer f.mutex.Unlock()
@ -553,9 +600,9 @@ func (f *FileActionTaskManager) fileActionTaskExecutor(ctx context.Context) {
f.setExecuteLoopFlag(true)
logger.Verboseln("file execute task is finish, exit normally")
prompt := ""
if f.task.Mode == UploadOnly {
if f.task.Mode == Upload {
prompt = "完成全部文件的同步上传，等待下一次扫描"
} else if f.task.Mode == DownloadOnly {
} else if f.task.Mode == Download {
prompt = "完成全部文件的同步下载，等待下一次扫描"
} else {
prompt = "完成全部文件的同步，等待下一次扫描"

59
internal/syncdrive/sync_task.go
View File

@ -20,9 +20,9 @@ import (
)
type (
TaskStep string
SyncMode string
CycleMode string
SyncMode string
SyncPolicy string
CycleMode string
// SyncTask 同步任务
SyncTask struct {
@ -40,8 +40,10 @@ type (
LocalFolderPath string `json:"localFolderPath"`
// PanFolderPath 云盘目录
PanFolderPath string `json:"panFolderPath"`
// Mode 同步模式
// Mode 备份模式
Mode SyncMode `json:"mode"`
// Policy 备份策略
Policy SyncPolicy `json:"policy"`
// CycleMode 循环模式，OneTime-运行一次，InfiniteLoop-无限循环模式
CycleModeType CycleMode `json:"-"`
// Priority 优先级选项
@ -73,24 +75,22 @@ type (
)
const (
// UploadOnly 单向上传，即备份本地文件到云盘
UploadOnly SyncMode = "upload"
// DownloadOnly 只下载，即备份云盘文件到本地
DownloadOnly SyncMode = "download"
// Upload 上传，即备份本地文件到云盘
Upload SyncMode = "upload"
// Download 下载，即备份云盘文件到本地
Download SyncMode = "download"
// SyncTwoWay 双向同步，本地和云盘文件完全保持一致
SyncTwoWay SyncMode = "sync"
// SyncPolicyExclusive 备份策略，排他备份，保证本地和云盘一比一备份，目标目录多余的文件会被删除
SyncPolicyExclusive SyncPolicy = "exclusive"
// SyncPolicyIncrement 备份策略，增量备份，只会增量备份文件，目标目录多余的（旧的）文件不会被删除
SyncPolicyIncrement SyncPolicy = "increment"
// CycleOneTime 只运行一次
CycleOneTime CycleMode = "OneTime"
// CycleInfiniteLoop 无限循环模式
CycleInfiniteLoop CycleMode = "InfiniteLoop"
// StepScanFile 任务步骤，扫描文件建立同步数据库
StepScanFile TaskStep = "scan"
// StepDiffFile 任务步骤，对比文件
StepDiffFile TaskStep = "diff"
// StepSyncFile 任务步骤，同步文件
StepSyncFile TaskStep = "sync"
)
func (t *SyncTask) NameLabel() string {
@ -101,11 +101,11 @@ func (t *SyncTask) String() string {
builder := &strings.Builder{}
builder.WriteString("任务: " + t.NameLabel() + "\n")
mode := "双向备份"
if t.Mode == UploadOnly {
mode = "备份本地文件（只上传）"
if t.Mode == Upload {
mode = "备份本地文件（上传）"
}
if t.Mode == DownloadOnly {
mode = "备份云盘文件（只下载）"
if t.Mode == Download {
mode = "备份云盘文件（下载）"
}
builder.WriteString("同步模式: " + mode + "\n")
if t.Mode == SyncTwoWay {
@ -119,6 +119,14 @@ func (t *SyncTask) String() string {
}
builder.WriteString("优先选项: " + priority + "\n")
}
policy := "增量备份"
if t.Policy == SyncPolicyExclusive {
policy = "排他备份（上传&删除）"
}
if t.Policy == SyncPolicyIncrement {
policy = "增量备份（只上传）"
}
builder.WriteString("同步策略: " + policy + "\n")
builder.WriteString("本地目录: " + t.LocalFolderPath + "\n")
builder.WriteString("云盘目录: " + t.PanFolderPath + "\n")
driveName := "备份盘"
@ -209,11 +217,16 @@ func (t *SyncTask) Start() error {
return e
}
// 策略
if t.Policy == "" {
t.Policy = SyncPolicyIncrement
}
// 启动文件扫描进程
t.SetScanLoopFlag(false)
if t.Mode == UploadOnly {
if t.Mode == Upload {
go t.scanLocalFile(t.ctx)
} else if t.Mode == DownloadOnly {
} else if t.Mode == Download {
go t.scanPanFile(t.ctx)
} else {
return fmt.Errorf("异常：暂不支持该模式。")
@ -333,7 +346,7 @@ func (t *SyncTask) discardLocalFileDb(filePath string, startTimeUnix int64) bool
}
for _, file := range files {
if file.ScanTimeAt == "" || file.ScanTimeUnix() < startTimeUnix {
if t.Mode == DownloadOnly {
if t.Mode == Download {
// delete discard local file info directly
t.localFileDb.Delete(file.Path)
logger.Verboseln("label discard local file from DB: ", utils.ObjectToJsonStr(file, false))
@ -572,7 +585,7 @@ func (t *SyncTask) discardPanFileDb(filePath string, startTimeUnix int64) bool {
}
for _, file := range files {
if file.ScanTimeUnix() < startTimeUnix {
if t.Mode == UploadOnly {
if t.Mode == Upload {
// delete discard pan file info directly
t.panFileDb.Delete(file.Path)
logger.Verboseln("delete discard pan file from DB: ", utils.ObjectToJsonStr(file, false))

3
internal/syncdrive/sync_task_mgr.go
View File

@ -180,6 +180,9 @@ func (m *SyncTaskManager) Start(tasks []*SyncTask) (bool, error) {
task.Priority = SyncPriorityTimestampFirst
task.syncOption.SyncPriority = SyncPriorityTimestampFirst
}
if task.Policy == "" {
task.Policy = SyncPolicyIncrement
}
task.LocalFolderPath = path.Clean(task.LocalFolderPath)
task.PanFolderPath = path.Clean(task.PanFolderPath)
if e := task.Start(); e != nil {