add sync file record #206

docker/sync/app.sh

@@ -39,5 +39,13 @@ else
   ./aliyunpan login -RefreshToken=${ALIYUNPAN_REFRESH_TOKEN}
 fi
 
-./aliyunpan config set -transfer_url_type ${ALIYUNPAN_TRANSFER_URL_TYPE} 
+./aliyunpan config set -transfer_url_type ${ALIYUNPAN_TRANSFER_URL_TYPE}
+
+if [ "$ALIYUNPAN_SYNC_LOG" = "true" ]
+then
+  ./aliyunpan config set -file_record_config 1
+else
+  ./aliyunpan config set -file_record_config 2
+fi
+
 ./aliyunpan sync start -dp ${ALIYUNPAN_DOWNLOAD_PARALLEL} -up ${ALIYUNPAN_UPLOAD_PARALLEL} -dbs ${ALIYUNPAN_DOWNLOAD_BLOCK_SIZE} -ubs ${ALIYUNPAN_UPLOAD_BLOCK_SIZE} -log ${ALIYUNPAN_SYNC_LOG} -ldt ${ALIYUNPAN_LOCAL_DELAY_TIME} -step ${ALIYUNPAN_TASK_STEP}

internal/command/sync.go

@@ -18,6 +18,7 @@ import (
 	"github.com/tickstep/aliyunpan-api/aliyunpan"
 	"github.com/tickstep/aliyunpan/cmder"
 	"github.com/tickstep/aliyunpan/internal/config"
+	"github.com/tickstep/aliyunpan/internal/log"
 	"github.com/tickstep/aliyunpan/internal/syncdrive"
 	"github.com/tickstep/aliyunpan/internal/utils"
 	"github.com/tickstep/library-go/converter"
@@ -318,6 +319,10 @@ func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadPa
 	if useInternalUrl {
 		typeUrlStr = "阿里ECS内部链接"
 	}
+
+	// 文件同步记录器
+	fileRecorder := log.NewFileRecorder(config.GetLogDir() + "/sync_file_records.csv")
+
 	option := syncdrive.SyncOption{
 		FileDownloadParallel:              fileDownloadParallel,
 		FileUploadParallel:                fileUploadParallel,
@@ -328,6 +333,7 @@ func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadPa
 		MaxUploadRate:                     maxUploadRate,
 		SyncPriority:                      flag,
 		LocalFileModifiedCheckIntervalSec: localDelayTime,
+		FileRecorder:                      fileRecorder,
 	}
 	syncMgr := syncdrive.NewSyncTaskManager(activeUser, activeUser.DriveList.GetFileDriveId(), panClient, syncFolderRootPath, option)
 	syncConfigFile := syncMgr.ConfigFilePath()

internal/syncdrive/file_action_task.go

@@ -5,10 +5,12 @@ import (
 	"fmt"
 	"github.com/tickstep/aliyunpan-api/aliyunpan"
 	"github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
+	"github.com/tickstep/aliyunpan/internal/config"
 	"github.com/tickstep/aliyunpan/internal/file/downloader"
 	"github.com/tickstep/aliyunpan/internal/file/uploader"
 	"github.com/tickstep/aliyunpan/internal/functions/panupload"
 	"github.com/tickstep/aliyunpan/internal/localfile"
+	"github.com/tickstep/aliyunpan/internal/log"
 	"github.com/tickstep/aliyunpan/internal/utils"
 	"github.com/tickstep/aliyunpan/library/requester/transfer"
 	"github.com/tickstep/library-go/converter"
@@ -39,6 +41,9 @@ type (
 
 		localFolderCreateMutex *sync.Mutex
 		panFolderCreateMutex   *sync.Mutex
+
+		// 文件记录器，存储同步文件记录
+		fileRecorder *log.FileRecorder
 	}
 )
 
@@ -68,12 +73,15 @@ func (f *FileActionTask) DoAction(ctx context.Context) error {
 		} else {
 			// upload success, post operation
 			// save local file info into db
+			var actFile *aliyunpan.FileEntity
 			if f.syncItem.UploadEntity != nil && f.syncItem.UploadEntity.FileId != "" {
 				if file, er := f.panClient.FileInfoById(f.syncItem.DriveId, f.syncItem.UploadEntity.FileId); er == nil {
 					file.Path = f.syncItem.getPanFileFullPath()
 					fItem := NewPanFileItem(file)
 					fItem.ScanTimeAt = utils.NowTimeStr()
 					f.panFileDb.Add(fItem)
+
+					actFile = file
 				}
 			} else {
 				if file, er := f.panClient.FileInfoByPath(f.syncItem.DriveId, f.syncItem.getPanFileFullPath()); er == nil {
@@ -81,8 +89,18 @@ func (f *FileActionTask) DoAction(ctx context.Context) error {
 					fItem := NewPanFileItem(file)
 					fItem.ScanTimeAt = utils.NowTimeStr()
 					f.panFileDb.Add(fItem)
+
+					actFile = file
 				}
 			}
+
+			// recorder
+			f.appendRecord(&log.FileRecordItem{
+				Status:   "成功-上传",
+				TimeStr:  utils.NowTimeStr(),
+				FileSize: actFile.FileSize,
+				FilePath: actFile.Path,
+			})
 		}
 	}
 
@@ -137,6 +155,14 @@ func (f *FileActionTask) DoAction(ctx context.Context) error {
 					ScanStatus:    ScanStatusNormal,
 				})
 			}
+
+			// recorder
+			f.appendRecord(&log.FileRecordItem{
+				Status:   "成功-下载",
+				TimeStr:  utils.NowTimeStr(),
+				FileSize: f.syncItem.PanFile.FileSize,
+				FilePath: f.syncItem.PanFile.Path,
+			})
 		}
 	}
 
@@ -149,6 +175,14 @@ func (f *FileActionTask) DoAction(ctx context.Context) error {
 			// clear DB
 			f.localFileDb.Delete(f.syncItem.getLocalFileFullPath())
 			f.panFileDb.Delete(f.syncItem.getPanFileFullPath())
+
+			// recorder
+			f.appendRecord(&log.FileRecordItem{
+				Status:   "成功-删除本地文件",
+				TimeStr:  utils.NowTimeStr(),
+				FileSize: 0,
+				FilePath: f.syncItem.getLocalFileFullPath(),
+			})
 		}
 	}
 
@@ -161,6 +195,14 @@ func (f *FileActionTask) DoAction(ctx context.Context) error {
 			// clear DB
 			f.localFileDb.Delete(f.syncItem.getLocalFileFullPath())
 			f.panFileDb.Delete(f.syncItem.getPanFileFullPath())
+
+			// recorder
+			f.appendRecord(&log.FileRecordItem{
+				Status:   "成功-删除云盘文件",
+				TimeStr:  utils.NowTimeStr(),
+				FileSize: 0,
+				FilePath: f.syncItem.getPanFileFullPath(),
+			})
 		}
 	}
 
@@ -716,3 +758,13 @@ func (f *FileActionTask) createPanFolder(ctx context.Context) error {
 		return apierr1
 	}
 }
+
+func (f *FileActionTask) appendRecord(item *log.FileRecordItem) error {
+	if item == nil {
+		return nil
+	}
+	if config.Config.FileRecordConfig == "1" {
+		return f.fileRecorder.Append(item)
+	}
+	return nil
+}

internal/syncdrive/file_action_task_mgr.go

@@ -729,6 +729,7 @@ func (f *FileActionTaskManager) getFromSyncDb(act SyncFileAction) *FileActionTas
 						maxUploadRate:          f.syncOption.MaxUploadRate,
 						localFolderCreateMutex: f.localCreateMutex,
 						panFolderCreateMutex:   f.folderCreateMutex,
+						fileRecorder:           f.syncOption.FileRecorder,
 					}
 				}
 			}
@@ -747,6 +748,7 @@ func (f *FileActionTaskManager) getFromSyncDb(act SyncFileAction) *FileActionTas
 						maxUploadRate:          f.syncOption.MaxUploadRate,
 						localFolderCreateMutex: f.localCreateMutex,
 						panFolderCreateMutex:   f.folderCreateMutex,
+						fileRecorder:           f.syncOption.FileRecorder,
 					}
 				}
 			}
@@ -767,6 +769,7 @@ func (f *FileActionTaskManager) getFromSyncDb(act SyncFileAction) *FileActionTas
 						maxUploadRate:          f.syncOption.MaxUploadRate,
 						localFolderCreateMutex: f.localCreateMutex,
 						panFolderCreateMutex:   f.folderCreateMutex,
+						fileRecorder:           f.syncOption.FileRecorder,
 					}
 				}
 			}

internal/syncdrive/sync_task_mgr.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/tickstep/aliyunpan-api/aliyunpan"
 	"github.com/tickstep/aliyunpan/internal/config"
+	"github.com/tickstep/aliyunpan/internal/log"
 	"github.com/tickstep/aliyunpan/internal/utils"
 	"github.com/tickstep/library-go/logger"
 	"io/ioutil"
@@ -29,6 +30,9 @@ type (
 
 		// 本地文件修改检测间隔
 		LocalFileModifiedCheckIntervalSec int
+
+		// 文件记录器
+		FileRecorder *log.FileRecorder
 	}
 
 	// SyncTaskManager 同步任务管理器