Clone/aliyunpan
1
0
Fork 0
mirror of https://github.com/tickstep/aliyunpan.git synced 2025-01-23 14:32:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add sync file record #206

Browse Source
This commit is contained in:
xiaoyaofenfen 2022-12-19 20:41:05 +08:00
parent c5d15b058a
commit 07746fc15c
5 changed files with 74 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docker/sync/app.sh
View File

@ -39,5 +39,13 @@ else
./aliyunpan login -RefreshToken=${ALIYUNPAN_REFRESH_TOKEN}
fi
./aliyunpan config set -transfer_url_type ${ALIYUNPAN_TRANSFER_URL_TYPE}
./aliyunpan config set -transfer_url_type ${ALIYUNPAN_TRANSFER_URL_TYPE}
if [ "$ALIYUNPAN_SYNC_LOG" = "true" ]
then
./aliyunpan config set -file_record_config 1
else
./aliyunpan config set -file_record_config 2
fi
./aliyunpan sync start -dp ${ALIYUNPAN_DOWNLOAD_PARALLEL} -up ${ALIYUNPAN_UPLOAD_PARALLEL} -dbs ${ALIYUNPAN_DOWNLOAD_BLOCK_SIZE} -ubs ${ALIYUNPAN_UPLOAD_BLOCK_SIZE} -log ${ALIYUNPAN_SYNC_LOG} -ldt ${ALIYUNPAN_LOCAL_DELAY_TIME} -step ${ALIYUNPAN_TASK_STEP}

6
internal/command/sync.go
View File

@ -18,6 +18,7 @@ import (
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan/cmder"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/aliyunpan/internal/log"
"github.com/tickstep/aliyunpan/internal/syncdrive"
"github.com/tickstep/aliyunpan/internal/utils"
"github.com/tickstep/library-go/converter"
@ -318,6 +319,10 @@ func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadPa
if useInternalUrl {
typeUrlStr = "阿里ECS内部链接"
}
// 文件同步记录器
fileRecorder := log.NewFileRecorder(config.GetLogDir() + "/sync_file_records.csv")
option := syncdrive.SyncOption{
FileDownloadParallel: fileDownloadParallel,
FileUploadParallel: fileUploadParallel,
@ -328,6 +333,7 @@ func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadPa
MaxUploadRate: maxUploadRate,
SyncPriority: flag,
LocalFileModifiedCheckIntervalSec: localDelayTime,
FileRecorder: fileRecorder,
}
syncMgr := syncdrive.NewSyncTaskManager(activeUser, activeUser.DriveList.GetFileDriveId(), panClient, syncFolderRootPath, option)
syncConfigFile := syncMgr.ConfigFilePath()

52
internal/syncdrive/file_action_task.go
View File

@ -5,10 +5,12 @@ import (
"fmt"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/aliyunpan/internal/file/downloader"
"github.com/tickstep/aliyunpan/internal/file/uploader"
"github.com/tickstep/aliyunpan/internal/functions/panupload"
"github.com/tickstep/aliyunpan/internal/localfile"
"github.com/tickstep/aliyunpan/internal/log"
"github.com/tickstep/aliyunpan/internal/utils"
"github.com/tickstep/aliyunpan/library/requester/transfer"
"github.com/tickstep/library-go/converter"
@ -39,6 +41,9 @@ type (
localFolderCreateMutex *sync.Mutex
panFolderCreateMutex *sync.Mutex
// 文件记录器，存储同步文件记录
fileRecorder *log.FileRecorder
}
)
@ -68,12 +73,15 @@ func (f *FileActionTask) DoAction(ctx context.Context) error {
} else {
// upload success, post operation
// save local file info into db
var actFile *aliyunpan.FileEntity
if f.syncItem.UploadEntity != nil && f.syncItem.UploadEntity.FileId != "" {
if file, er := f.panClient.FileInfoById(f.syncItem.DriveId, f.syncItem.UploadEntity.FileId); er == nil {
file.Path = f.syncItem.getPanFileFullPath()
fItem := NewPanFileItem(file)
fItem.ScanTimeAt = utils.NowTimeStr()
f.panFileDb.Add(fItem)
actFile = file
}
} else {
if file, er := f.panClient.FileInfoByPath(f.syncItem.DriveId, f.syncItem.getPanFileFullPath()); er == nil {
@ -81,8 +89,18 @@ func (f *FileActionTask) DoAction(ctx context.Context) error {
fItem := NewPanFileItem(file)
fItem.ScanTimeAt = utils.NowTimeStr()
f.panFileDb.Add(fItem)
actFile = file
}
}
// recorder
f.appendRecord(&log.FileRecordItem{
Status: "成功-上传",
TimeStr: utils.NowTimeStr(),
FileSize: actFile.FileSize,
FilePath: actFile.Path,
})
}
}
@ -137,6 +155,14 @@ func (f *FileActionTask) DoAction(ctx context.Context) error {
ScanStatus: ScanStatusNormal,
})
}
// recorder
f.appendRecord(&log.FileRecordItem{
Status: "成功-下载",
TimeStr: utils.NowTimeStr(),
FileSize: f.syncItem.PanFile.FileSize,
FilePath: f.syncItem.PanFile.Path,
})
}
}
@ -149,6 +175,14 @@ func (f *FileActionTask) DoAction(ctx context.Context) error {
// clear DB
f.localFileDb.Delete(f.syncItem.getLocalFileFullPath())
f.panFileDb.Delete(f.syncItem.getPanFileFullPath())
// recorder
f.appendRecord(&log.FileRecordItem{
Status: "成功-删除本地文件",
TimeStr: utils.NowTimeStr(),
FileSize: 0,
FilePath: f.syncItem.getLocalFileFullPath(),
})
}
}
@ -161,6 +195,14 @@ func (f *FileActionTask) DoAction(ctx context.Context) error {
// clear DB
f.localFileDb.Delete(f.syncItem.getLocalFileFullPath())
f.panFileDb.Delete(f.syncItem.getPanFileFullPath())
// recorder
f.appendRecord(&log.FileRecordItem{
Status: "成功-删除云盘文件",
TimeStr: utils.NowTimeStr(),
FileSize: 0,
FilePath: f.syncItem.getPanFileFullPath(),
})
}
}
@ -716,3 +758,13 @@ func (f *FileActionTask) createPanFolder(ctx context.Context) error {
return apierr1
}
}
func (f *FileActionTask) appendRecord(item *log.FileRecordItem) error {
if item == nil {
return nil
}
if config.Config.FileRecordConfig == "1" {
return f.fileRecorder.Append(item)
}
return nil
}

3
internal/syncdrive/file_action_task_mgr.go
View File

@ -729,6 +729,7 @@ func (f *FileActionTaskManager) getFromSyncDb(act SyncFileAction) *FileActionTas
maxUploadRate: f.syncOption.MaxUploadRate,
localFolderCreateMutex: f.localCreateMutex,
panFolderCreateMutex: f.folderCreateMutex,
fileRecorder: f.syncOption.FileRecorder,
}
}
}
@ -747,6 +748,7 @@ func (f *FileActionTaskManager) getFromSyncDb(act SyncFileAction) *FileActionTas
maxUploadRate: f.syncOption.MaxUploadRate,
localFolderCreateMutex: f.localCreateMutex,
panFolderCreateMutex: f.folderCreateMutex,
fileRecorder: f.syncOption.FileRecorder,
}
}
}
@ -767,6 +769,7 @@ func (f *FileActionTaskManager) getFromSyncDb(act SyncFileAction) *FileActionTas
maxUploadRate: f.syncOption.MaxUploadRate,
localFolderCreateMutex: f.localCreateMutex,
panFolderCreateMutex: f.folderCreateMutex,
fileRecorder: f.syncOption.FileRecorder,
}
}
}

4
internal/syncdrive/sync_task_mgr.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/aliyunpan/internal/log"
"github.com/tickstep/aliyunpan/internal/utils"
"github.com/tickstep/library-go/logger"
"io/ioutil"
@ -29,6 +30,9 @@ type (
// 本地文件修改检测间隔
LocalFileModifiedCheckIntervalSec int
// 文件记录器
FileRecorder *log.FileRecorder
}
// SyncTaskManager 同步任务管理器