mirror of
https://github.com/tickstep/aliyunpan.git
synced 2025-01-24 06:52:16 +08:00
611 lines
17 KiB
Go
611 lines
17 KiB
Go
package syncdrive
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
mapset "github.com/deckarep/golang-set"
|
||
"github.com/tickstep/aliyunpan/internal/localfile"
|
||
"github.com/tickstep/aliyunpan/internal/waitgroup"
|
||
"github.com/tickstep/aliyunpan/library/collection"
|
||
"github.com/tickstep/library-go/logger"
|
||
"path"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
type (
|
||
FileActionTaskList []*FileActionTask
|
||
|
||
FileActionTaskManager struct {
|
||
mutex *sync.Mutex
|
||
|
||
task *SyncTask
|
||
wg *waitgroup.WaitGroup
|
||
ctx context.Context
|
||
cancelFunc context.CancelFunc
|
||
|
||
fileInProcessQueue *collection.Queue
|
||
fileDownloadParallel int
|
||
fileUploadParallel int
|
||
}
|
||
|
||
localFileSet struct {
|
||
items LocalFileList
|
||
localFolderPath string
|
||
}
|
||
panFileSet struct {
|
||
items PanFileList
|
||
panFolderPath string
|
||
}
|
||
)
|
||
|
||
func NewFileActionTaskManager(task *SyncTask) *FileActionTaskManager {
|
||
return &FileActionTaskManager{
|
||
mutex: &sync.Mutex{},
|
||
task: task,
|
||
|
||
fileInProcessQueue: collection.NewFifoQueue(),
|
||
fileDownloadParallel: 2,
|
||
fileUploadParallel: 2,
|
||
}
|
||
}
|
||
|
||
func (f *FileActionTaskManager) Start() error {
|
||
if f.ctx != nil {
|
||
return fmt.Errorf("task have starting")
|
||
}
|
||
f.wg = waitgroup.NewWaitGroup(0)
|
||
|
||
var cancel context.CancelFunc
|
||
f.ctx, cancel = context.WithCancel(context.Background())
|
||
f.cancelFunc = cancel
|
||
|
||
go f.doLocalFileDiffRoutine(f.ctx)
|
||
go f.doPanFileDiffRoutine(f.ctx)
|
||
go f.fileActionTaskExecutor(f.ctx)
|
||
return nil
|
||
}
|
||
|
||
func (f *FileActionTaskManager) Stop() error {
|
||
if f.ctx == nil {
|
||
return nil
|
||
}
|
||
// cancel all sub task & process
|
||
f.cancelFunc()
|
||
|
||
// wait for finished
|
||
f.wg.Wait()
|
||
|
||
f.ctx = nil
|
||
f.cancelFunc = nil
|
||
|
||
return nil
|
||
}
|
||
|
||
// getPanPathFromLocalPath 通过本地文件路径获取网盘文件的对应路径
|
||
func (f *FileActionTaskManager) getPanPathFromLocalPath(localPath string) string {
|
||
localPath = strings.ReplaceAll(localPath, "\\", "/")
|
||
localRootPath := strings.ReplaceAll(f.task.LocalFolderPath, "\\", "/")
|
||
|
||
relativePath := strings.TrimPrefix(localPath, localRootPath)
|
||
return path.Join(path.Clean(f.task.PanFolderPath), relativePath)
|
||
}
|
||
|
||
// getLocalPathFromPanPath 通过网盘文件路径获取对应的本地文件的对应路径
|
||
func (f *FileActionTaskManager) getLocalPathFromPanPath(panPath string) string {
|
||
panPath = strings.ReplaceAll(panPath, "\\", "/")
|
||
panRootPath := strings.ReplaceAll(f.task.PanFolderPath, "\\", "/")
|
||
|
||
relativePath := strings.TrimPrefix(panPath, panRootPath)
|
||
return path.Join(path.Clean(f.task.LocalFolderPath), relativePath)
|
||
}
|
||
|
||
// doLocalFileDiffRoutine 对比网盘文件和本地文件信息，差异化上传或者下载文件
|
||
func (f *FileActionTaskManager) doLocalFileDiffRoutine(ctx context.Context) {
|
||
localFolderQueue := collection.NewFifoQueue()
|
||
|
||
// init root folder
|
||
localRootFolder, e := f.task.localFileDb.Get(f.task.LocalFolderPath)
|
||
if e == nil {
|
||
localFolderQueue.Push(localRootFolder)
|
||
} else {
|
||
logger.Verboseln(e)
|
||
return
|
||
}
|
||
|
||
f.wg.AddDelta()
|
||
defer f.wg.Done()
|
||
for {
|
||
select {
|
||
case <-ctx.Done():
|
||
// cancel routine & done
|
||
logger.Verboseln("file diff routine done")
|
||
return
|
||
default:
|
||
logger.Verboseln("do file diff process")
|
||
localFiles := LocalFileList{}
|
||
panFiles := PanFileList{}
|
||
var err error
|
||
var objLocal interface{}
|
||
|
||
objLocal = localFolderQueue.Pop()
|
||
if objLocal == nil {
|
||
// restart over
|
||
localFolderQueue.Push(localRootFolder)
|
||
time.Sleep(3 * time.Second)
|
||
continue
|
||
}
|
||
localItem := objLocal.(*LocalFileItem)
|
||
localFiles, err = f.task.localFileDb.GetFileList(localItem.Path)
|
||
if err != nil {
|
||
localFiles = LocalFileList{}
|
||
}
|
||
panFiles, err = f.task.panFileDb.GetFileList(f.getPanPathFromLocalPath(localItem.Path))
|
||
if err != nil {
|
||
panFiles = PanFileList{}
|
||
}
|
||
f.doFileDiffRoutine(panFiles, localFiles, nil, localFolderQueue)
|
||
}
|
||
}
|
||
}
|
||
|
||
// doPanFileDiffRoutine 对比网盘文件和本地文件信息，差异化上传或者下载文件
|
||
func (f *FileActionTaskManager) doPanFileDiffRoutine(ctx context.Context) {
|
||
panFolderQueue := collection.NewFifoQueue()
|
||
|
||
// init root folder
|
||
panRootFolder, e := f.task.panFileDb.Get(f.task.PanFolderPath)
|
||
if e == nil {
|
||
panFolderQueue.Push(panRootFolder)
|
||
} else {
|
||
logger.Verboseln(e)
|
||
return
|
||
}
|
||
|
||
f.wg.AddDelta()
|
||
defer f.wg.Done()
|
||
for {
|
||
select {
|
||
case <-ctx.Done():
|
||
// cancel routine & done
|
||
logger.Verboseln("file diff routine done")
|
||
return
|
||
default:
|
||
logger.Verboseln("do file diff process")
|
||
localFiles := LocalFileList{}
|
||
panFiles := PanFileList{}
|
||
var err error
|
||
var objPan interface{}
|
||
|
||
objPan = panFolderQueue.Pop()
|
||
if objPan == nil {
|
||
// restart over
|
||
panFolderQueue.Push(panRootFolder)
|
||
time.Sleep(3 * time.Second)
|
||
continue
|
||
}
|
||
panItem := objPan.(*PanFileItem)
|
||
panFiles, err = f.task.panFileDb.GetFileList(panItem.Path)
|
||
if err != nil {
|
||
panFiles = PanFileList{}
|
||
}
|
||
localFiles, err = f.task.localFileDb.GetFileList(f.getLocalPathFromPanPath(panItem.Path))
|
||
if err != nil {
|
||
localFiles = LocalFileList{}
|
||
}
|
||
f.doFileDiffRoutine(panFiles, localFiles, panFolderQueue, nil)
|
||
time.Sleep(500 * time.Millisecond)
|
||
}
|
||
}
|
||
}
|
||
|
||
func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFiles LocalFileList, panFolderQueue *collection.Queue, localFolderQueue *collection.Queue) {
|
||
// empty loop
|
||
if len(panFiles) == 0 && len(localFiles) == 0 {
|
||
time.Sleep(100 * time.Millisecond)
|
||
return
|
||
}
|
||
|
||
localFilesSet := &localFileSet{
|
||
items: localFiles,
|
||
localFolderPath: f.task.LocalFolderPath,
|
||
}
|
||
panFilesSet := &panFileSet{
|
||
items: panFiles,
|
||
panFolderPath: f.task.PanFolderPath,
|
||
}
|
||
localFilesNeedToUpload := localFilesSet.Difference(panFilesSet)
|
||
panFilesNeedToDownload := panFilesSet.Difference(localFilesSet)
|
||
localFilesNeedToCheck, panFilesNeedToCheck := localFilesSet.Intersection(panFilesSet)
|
||
|
||
// download file from pan drive
|
||
if f.task.Mode != UploadOnly {
|
||
for _, file := range panFilesNeedToDownload {
|
||
if file.IsFolder() {
|
||
if panFolderQueue != nil {
|
||
panFolderQueue.PushUnique(file)
|
||
}
|
||
continue
|
||
}
|
||
fileActionTask := &FileActionTask{
|
||
syncItem: &SyncFileItem{
|
||
Action: SyncFileActionDownload,
|
||
Status: SyncFileStatusCreate,
|
||
LocalFile: nil,
|
||
PanFile: file,
|
||
StatusUpdateTime: "",
|
||
PanFolderPath: f.task.PanFolderPath,
|
||
LocalFolderPath: f.task.LocalFolderPath,
|
||
},
|
||
}
|
||
f.addToSyncDb(fileActionTask)
|
||
}
|
||
}
|
||
|
||
// upload file to pan drive
|
||
if f.task.Mode != DownloadOnly {
|
||
for _, file := range localFilesNeedToUpload {
|
||
if file.IsFolder() {
|
||
if localFolderQueue != nil {
|
||
localFolderQueue.PushUnique(file)
|
||
}
|
||
continue
|
||
}
|
||
fileActionTask := &FileActionTask{
|
||
syncItem: &SyncFileItem{
|
||
Action: SyncFileActionUpload,
|
||
Status: SyncFileStatusCreate,
|
||
LocalFile: file,
|
||
PanFile: nil,
|
||
StatusUpdateTime: "",
|
||
PanFolderPath: f.task.PanFolderPath,
|
||
LocalFolderPath: f.task.LocalFolderPath,
|
||
},
|
||
}
|
||
f.addToSyncDb(fileActionTask)
|
||
}
|
||
}
|
||
|
||
// compare file to decide download / upload
|
||
for idx, _ := range localFilesNeedToCheck {
|
||
localFile := localFilesNeedToCheck[idx]
|
||
panFile := panFilesNeedToCheck[idx]
|
||
if localFile.IsFolder() {
|
||
if localFolderQueue != nil {
|
||
localFolderQueue.PushUnique(localFile)
|
||
}
|
||
if panFolderQueue != nil {
|
||
panFolderQueue.PushUnique(panFile)
|
||
}
|
||
continue
|
||
}
|
||
|
||
if localFile.Sha1Hash == "" {
|
||
// calc sha1
|
||
fileSum := localfile.NewLocalFileEntity(localFile.Path)
|
||
fileSum.Sum(localfile.CHECKSUM_SHA1) // block operation
|
||
localFile.Sha1Hash = fileSum.SHA1
|
||
fileSum.Close()
|
||
|
||
// save sha1
|
||
f.task.localFileDb.Update(localFile)
|
||
}
|
||
|
||
if strings.ToLower(panFile.Sha1Hash) == strings.ToLower(localFile.Sha1Hash) {
|
||
// do nothing
|
||
logger.Verboseln("no need to update file: ", localFile.Path)
|
||
continue
|
||
}
|
||
|
||
// 本地文件和云盘文件SHA1不一样
|
||
// 不同模式同步策略不一样
|
||
if f.task.Mode == UploadOnly {
|
||
uploadLocalFile := &FileActionTask{
|
||
syncItem: &SyncFileItem{
|
||
Action: SyncFileActionUpload,
|
||
Status: SyncFileStatusCreate,
|
||
LocalFile: localFile,
|
||
PanFile: nil,
|
||
StatusUpdateTime: "",
|
||
PanFolderPath: f.task.PanFolderPath,
|
||
LocalFolderPath: f.task.LocalFolderPath,
|
||
},
|
||
}
|
||
f.addToSyncDb(uploadLocalFile)
|
||
} else if f.task.Mode == DownloadOnly {
|
||
downloadPanFile := &FileActionTask{
|
||
syncItem: &SyncFileItem{
|
||
Action: SyncFileActionDownload,
|
||
Status: SyncFileStatusCreate,
|
||
LocalFile: nil,
|
||
PanFile: panFile,
|
||
StatusUpdateTime: "",
|
||
PanFolderPath: f.task.PanFolderPath,
|
||
LocalFolderPath: f.task.LocalFolderPath,
|
||
},
|
||
}
|
||
f.addToSyncDb(downloadPanFile)
|
||
} else if f.task.Mode == SyncTwoWay {
|
||
if localFile.UpdateTimeUnix() > panFile.UpdateTimeUnix() { // upload file
|
||
uploadLocalFile := &FileActionTask{
|
||
syncItem: &SyncFileItem{
|
||
Action: SyncFileActionUpload,
|
||
Status: SyncFileStatusCreate,
|
||
LocalFile: localFile,
|
||
PanFile: nil,
|
||
StatusUpdateTime: "",
|
||
PanFolderPath: f.task.PanFolderPath,
|
||
LocalFolderPath: f.task.LocalFolderPath,
|
||
},
|
||
}
|
||
f.addToSyncDb(uploadLocalFile)
|
||
} else if localFile.UpdateTimeUnix() < panFile.UpdateTimeUnix() { // download file
|
||
downloadPanFile := &FileActionTask{
|
||
syncItem: &SyncFileItem{
|
||
Action: SyncFileActionDownload,
|
||
Status: SyncFileStatusCreate,
|
||
LocalFile: nil,
|
||
PanFile: panFile,
|
||
StatusUpdateTime: "",
|
||
PanFolderPath: f.task.PanFolderPath,
|
||
LocalFolderPath: f.task.LocalFolderPath,
|
||
},
|
||
}
|
||
f.addToSyncDb(downloadPanFile)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func (f *FileActionTaskManager) addToSyncDb(fileTask *FileActionTask) {
|
||
f.mutex.Lock()
|
||
defer f.mutex.Unlock()
|
||
|
||
// sync scan time
|
||
//if fileTask.syncItem.Action == SyncFileActionDownload {
|
||
// if (time.Now().Unix() - fileTask.syncItem.PanFile.ScanTimeUnix()) > TimeSecondsOf30Minute {
|
||
// // 大于30分钟，不同步，文件信息可能已经过期
|
||
// return
|
||
// }
|
||
//}
|
||
|
||
// check sync db
|
||
if itemInDb, e := f.task.syncFileDb.Get(fileTask.syncItem.Id()); e == nil && itemInDb != nil {
|
||
if itemInDb.Status == SyncFileStatusCreate || itemInDb.Status == SyncFileStatusDownloading || itemInDb.Status == SyncFileStatusUploading {
|
||
return
|
||
}
|
||
if itemInDb.Status == SyncFileStatusSuccess {
|
||
if (time.Now().Unix() - itemInDb.StatusUpdateTimeUnix()) < TimeSecondsOf5Minute {
|
||
// 少于5分钟，不同步，减少同步频次
|
||
return
|
||
}
|
||
}
|
||
if itemInDb.Status == SyncFileStatusIllegal {
|
||
if (time.Now().Unix() - itemInDb.StatusUpdateTimeUnix()) < TimeSecondsOf60Minute {
|
||
// 非法文件，少于60分钟，不同步，减少同步频次
|
||
return
|
||
}
|
||
}
|
||
if itemInDb.Status == SyncFileStatusNotExisted {
|
||
if itemInDb.Action == SyncFileActionDownload {
|
||
if itemInDb.PanFile.UpdatedAt == fileTask.syncItem.PanFile.UpdatedAt {
|
||
return
|
||
}
|
||
} else if itemInDb.Action == SyncFileActionUpload {
|
||
if itemInDb.LocalFile.UpdatedAt == fileTask.syncItem.LocalFile.UpdatedAt {
|
||
return
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 进入任务队列
|
||
f.task.syncFileDb.Add(fileTask.syncItem)
|
||
}
|
||
|
||
func (f *FileActionTaskManager) getFromSyncDb(act SyncFileAction) *FileActionTask {
|
||
f.mutex.Lock()
|
||
defer f.mutex.Unlock()
|
||
|
||
if act == SyncFileActionDownload {
|
||
if files, e := f.task.syncFileDb.GetFileList(SyncFileStatusDownloading); e == nil {
|
||
for _, file := range files {
|
||
if !f.fileInProcessQueue.Contains(file) {
|
||
return &FileActionTask{
|
||
localFileDb: f.task.localFileDb,
|
||
panFileDb: f.task.panFileDb,
|
||
syncFileDb: f.task.syncFileDb,
|
||
panClient: f.task.panClient,
|
||
blockSize: int64(10485760),
|
||
syncItem: file,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else if act == SyncFileActionUpload {
|
||
if files, e := f.task.syncFileDb.GetFileList(SyncFileStatusUploading); e == nil {
|
||
for _, file := range files {
|
||
if !f.fileInProcessQueue.Contains(file) {
|
||
return &FileActionTask{
|
||
localFileDb: f.task.localFileDb,
|
||
panFileDb: f.task.panFileDb,
|
||
syncFileDb: f.task.syncFileDb,
|
||
panClient: f.task.panClient,
|
||
blockSize: int64(10485760),
|
||
syncItem: file,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if files, e := f.task.syncFileDb.GetFileList(SyncFileStatusCreate); e == nil {
|
||
if len(files) > 0 {
|
||
for _, file := range files {
|
||
if file.Action == act && !f.fileInProcessQueue.Contains(file) {
|
||
return &FileActionTask{
|
||
localFileDb: f.task.localFileDb,
|
||
panFileDb: f.task.panFileDb,
|
||
syncFileDb: f.task.syncFileDb,
|
||
panClient: f.task.panClient,
|
||
blockSize: int64(10485760),
|
||
syncItem: file,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// cleanSyncDbRecords 清楚同步数据库无用数据
|
||
func (f *FileActionTaskManager) cleanSyncDbRecords(ctx context.Context) {
|
||
// TODO: failed / success / illegal
|
||
}
|
||
|
||
// fileActionTaskExecutor 异步执行文件操作
|
||
func (f *FileActionTaskManager) fileActionTaskExecutor(ctx context.Context) {
|
||
f.wg.AddDelta()
|
||
defer f.wg.Done()
|
||
|
||
downloadWaitGroup := waitgroup.NewWaitGroup(f.fileDownloadParallel)
|
||
|
||
for {
|
||
select {
|
||
case <-ctx.Done():
|
||
// cancel routine & done
|
||
logger.Verboseln("file executor routine done")
|
||
downloadWaitGroup.Wait()
|
||
return
|
||
default:
|
||
logger.Verboseln("do file executor process")
|
||
|
||
// do upload
|
||
//uploadItem := f.getFromSyncDb(SyncFileActionUpload)
|
||
//if uploadItem != nil {
|
||
// f.fileInProcessQueue.Add(uploadItem)
|
||
// uploadItem.DoAction(ctx)
|
||
//}
|
||
|
||
// do download
|
||
downloadItem := f.getFromSyncDb(SyncFileActionDownload)
|
||
if downloadItem != nil {
|
||
if downloadWaitGroup.Parallel() < f.fileDownloadParallel {
|
||
downloadWaitGroup.AddDelta()
|
||
f.fileInProcessQueue.PushUnique(downloadItem.syncItem)
|
||
go func() {
|
||
if e := downloadItem.DoAction(ctx); e == nil {
|
||
// success
|
||
f.fileInProcessQueue.Remove(downloadItem)
|
||
} else {
|
||
// retry?
|
||
f.fileInProcessQueue.Remove(downloadItem)
|
||
}
|
||
downloadWaitGroup.Done()
|
||
}()
|
||
}
|
||
}
|
||
|
||
// delay
|
||
time.Sleep(500 * time.Millisecond)
|
||
}
|
||
}
|
||
}
|
||
|
||
// getRelativePath 获取文件的相对路径
|
||
func (l *localFileSet) getRelativePath(localPath string) string {
|
||
localPath = strings.ReplaceAll(localPath, "\\", "/")
|
||
localRootPath := strings.ReplaceAll(l.localFolderPath, "\\", "/")
|
||
relativePath := strings.TrimPrefix(localPath, localRootPath)
|
||
return path.Clean(relativePath)
|
||
}
|
||
|
||
// Intersection 交集
|
||
func (l *localFileSet) Intersection(other *panFileSet) (LocalFileList, PanFileList) {
|
||
localFilePathSet := mapset.NewThreadUnsafeSet()
|
||
relativePathLocalMap := map[string]*LocalFileItem{}
|
||
for _, item := range l.items {
|
||
rp := l.getRelativePath(item.Path)
|
||
relativePathLocalMap[rp] = item
|
||
localFilePathSet.Add(rp)
|
||
}
|
||
|
||
localFileList := LocalFileList{}
|
||
panFileList := PanFileList{}
|
||
for _, item := range other.items {
|
||
rp := other.getRelativePath(item.Path)
|
||
if localFilePathSet.Contains(rp) {
|
||
localFileList = append(localFileList, relativePathLocalMap[rp])
|
||
panFileList = append(panFileList, item)
|
||
}
|
||
}
|
||
return localFileList, panFileList
|
||
}
|
||
|
||
// Difference 差集
|
||
func (l *localFileSet) Difference(other *panFileSet) LocalFileList {
|
||
panFilePathSet := mapset.NewThreadUnsafeSet()
|
||
for _, item := range other.items {
|
||
rp := other.getRelativePath(item.Path)
|
||
panFilePathSet.Add(rp)
|
||
}
|
||
|
||
localFileList := LocalFileList{}
|
||
for _, item := range l.items {
|
||
rp := l.getRelativePath(item.Path)
|
||
if !panFilePathSet.Contains(rp) {
|
||
localFileList = append(localFileList, item)
|
||
}
|
||
}
|
||
return localFileList
|
||
}
|
||
|
||
// getRelativePath 获取文件的相对路径
|
||
func (p *panFileSet) getRelativePath(panPath string) string {
|
||
panPath = strings.ReplaceAll(panPath, "\\", "/")
|
||
panRootPath := strings.ReplaceAll(p.panFolderPath, "\\", "/")
|
||
relativePath := strings.TrimPrefix(panPath, panRootPath)
|
||
return path.Clean(relativePath)
|
||
}
|
||
|
||
// Intersection 交集
|
||
func (p *panFileSet) Intersection(other *localFileSet) (PanFileList, LocalFileList) {
|
||
localFilePathSet := mapset.NewThreadUnsafeSet()
|
||
relativePathLocalMap := map[string]*LocalFileItem{}
|
||
for _, item := range other.items {
|
||
rp := other.getRelativePath(item.Path)
|
||
relativePathLocalMap[rp] = item
|
||
localFilePathSet.Add(rp)
|
||
}
|
||
|
||
localFileList := LocalFileList{}
|
||
panFileList := PanFileList{}
|
||
for _, item := range p.items {
|
||
rp := p.getRelativePath(item.Path)
|
||
if localFilePathSet.Contains(rp) {
|
||
localFileList = append(localFileList, relativePathLocalMap[rp])
|
||
panFileList = append(panFileList, item)
|
||
}
|
||
}
|
||
return panFileList, localFileList
|
||
}
|
||
|
||
// Difference 差集
|
||
func (p *panFileSet) Difference(other *localFileSet) PanFileList {
|
||
localFilePathSet := mapset.NewThreadUnsafeSet()
|
||
for _, item := range other.items {
|
||
rp := other.getRelativePath(item.Path)
|
||
localFilePathSet.Add(rp)
|
||
}
|
||
|
||
panFileList := PanFileList{}
|
||
for _, item := range p.items {
|
||
rp := p.getRelativePath(item.Path)
|
||
if !localFilePathSet.Contains(rp) {
|
||
panFileList = append(panFileList, item)
|
||
}
|
||
}
|
||
return panFileList
|
||
}
|