From bb8dad726273ca5d47ccc6c17f0fccb1b5538d1c Mon Sep 17 00:00:00 2001 From: xiaoyaofenfen <1254525673@qq.com> Date: Wed, 21 Dec 2022 15:30:35 +0800 Subject: [PATCH] make download/upload/sync command singleton #208 #207 --- internal/command/download.go | 14 +++++ internal/command/sync.go | 14 +++++ internal/command/upload.go | 14 +++++ internal/config/pan_config.go | 5 ++ library/filelocker/errors.go | 7 +++ library/filelocker/file_locker.go | 25 ++++++++ library/filelocker/file_locker_test.go | 25 ++++++++ library/filelocker/locker_solaris.go | 57 +++++++++++++++++ library/filelocker/locker_unix.go | 50 +++++++++++++++ library/filelocker/locker_windows.go | 86 ++++++++++++++++++++++++++ 10 files changed, 297 insertions(+) create mode 100644 library/filelocker/errors.go create mode 100644 library/filelocker/file_locker.go create mode 100644 library/filelocker/file_locker_test.go create mode 100644 library/filelocker/locker_solaris.go create mode 100644 library/filelocker/locker_unix.go create mode 100644 library/filelocker/locker_windows.go diff --git a/internal/command/download.go b/internal/command/download.go index 8cfa0ef..b2e5dff 100644 --- a/internal/command/download.go +++ b/internal/command/download.go @@ -23,6 +23,7 @@ import ( "github.com/tickstep/aliyunpan/internal/log" "github.com/tickstep/aliyunpan/internal/taskframework" "github.com/tickstep/aliyunpan/internal/utils" + "github.com/tickstep/aliyunpan/library/filelocker" "github.com/tickstep/aliyunpan/library/requester/transfer" "github.com/tickstep/library-go/converter" "github.com/tickstep/library-go/logger" @@ -143,7 +144,20 @@ func CmdDownload() cli.Command { ExcludeNames: c.StringSlice("exn"), } + // 获取下载文件锁，保证下载操作单实例 + locker := filelocker.NewFileLocker(config.GetLockerDir() + "/aliyunpan-download") + if e := filelocker.LockFile(locker, 0755, true, 5*time.Second); e != nil { + logger.Verboseln(e) + fmt.Println("本应用其他实例正在执行下载，请先停止或者等待其完成") + return nil + } + RunDownload(c.Args(), do) + + // 释放文件锁 + if locker != nil { + filelocker.UnlockFile(locker) + } return nil }, Flags: []cli.Flag{ diff --git a/internal/command/sync.go b/internal/command/sync.go index a6ffccb..bd67719 100644 --- a/internal/command/sync.go +++ b/internal/command/sync.go @@ -21,6 +21,7 @@ import ( "github.com/tickstep/aliyunpan/internal/log" "github.com/tickstep/aliyunpan/internal/syncdrive" "github.com/tickstep/aliyunpan/internal/utils" + "github.com/tickstep/aliyunpan/library/filelocker" "github.com/tickstep/library-go/converter" "github.com/tickstep/library-go/logger" "github.com/urfave/cli" @@ -218,7 +219,20 @@ priority - 优先级，只对双向同步备份模式有效。选项支持三种 task.UserId = activeUser.UserId } + // 获取同步文件锁，保证同步操作单实例 + locker := filelocker.NewFileLocker(config.GetLockerDir() + "/aliyunpan-sync") + if e := filelocker.LockFile(locker, 0755, true, 5*time.Second); e != nil { + logger.Verboseln(e) + fmt.Println("本应用其他实例正在执行同步，请先停止或者等待其完成") + return nil + } + RunSync(task, dp, up, downloadBlockSize, uploadBlockSize, syncOpt, c.Int("ldt"), step) + + // 释放文件锁 + if locker != nil { + filelocker.UnlockFile(locker) + } return nil }, Flags: []cli.Flag{ diff --git a/internal/command/upload.go b/internal/command/upload.go index c6efff0..2e11e82 100644 --- a/internal/command/upload.go +++ b/internal/command/upload.go @@ -20,6 +20,7 @@ import ( "github.com/tickstep/aliyunpan/internal/log" "github.com/tickstep/aliyunpan/internal/plugins" "github.com/tickstep/aliyunpan/internal/utils" + "github.com/tickstep/aliyunpan/library/filelocker" "github.com/tickstep/library-go/requester/rio/speeds" "os" "path" @@ -176,6 +177,14 @@ func CmdUpload() cli.Command { } } + // 获取上传文件锁，保证上传操作单实例 + locker := filelocker.NewFileLocker(config.GetLockerDir() + "/aliyunpan-upload") + if e := filelocker.LockFile(locker, 0755, true, 5*time.Second); e != nil { + logger.Verboseln(e) + fmt.Println("本应用其他实例正在执行上传，请先停止或者等待其完成") + return nil + } + RunUpload(subArgs[:c.NArg()-1], subArgs[c.NArg()-1], &UploadOptions{ AllParallel: c.Int("p"), // 多文件上传的时候，允许同时并行上传的文件数量 Parallel: 1, // 一个文件同时多少个线程并发上传的数量。阿里云盘只支持单线程按顺序进行文件part数据上传，所以只能是1 @@ -188,6 +197,11 @@ func CmdUpload() cli.Command { ExcludeNames: c.StringSlice("exn"), BlockSize: int64(c.Int("bs") * 1024), }) + + // 释放文件锁 + if locker != nil { + filelocker.UnlockFile(locker) + } return nil }, Flags: UploadFlags, diff --git a/internal/config/pan_config.go b/internal/config/pan_config.go index 2441127..4c54759 100644 --- a/internal/config/pan_config.go +++ b/internal/config/pan_config.go @@ -351,6 +351,11 @@ func GetLogFilePath() string { return dirPath + "/" + "aliyunpan_verbose.log" } +// GetLockerDir 获取文件锁路径 +func GetLockerDir() string { + return strings.TrimSuffix(GetConfigDir(), "/") +} + func (c *PanConfig) ActiveUser() *PanUser { if c.activeUser == nil { if c.UserList == nil { diff --git a/library/filelocker/errors.go b/library/filelocker/errors.go new file mode 100644 index 0000000..98fee07 --- /dev/null +++ b/library/filelocker/errors.go @@ -0,0 +1,7 @@ +package filelocker + +import "errors" + +var ( + ErrTimeout = errors.New("timeout") +) diff --git a/library/filelocker/file_locker.go b/library/filelocker/file_locker.go new file mode 100644 index 0000000..77ec001 --- /dev/null +++ b/library/filelocker/file_locker.go @@ -0,0 +1,25 @@ +package filelocker + +import ( + "os" +) + +const ( + lockExt = ".lock" +) + +type ( + FileLocker struct { + FilePath string + LockFilePath string + lockFile *os.File + } +) + +func NewFileLocker(path string) *FileLocker { + return &FileLocker{ + FilePath: path, + LockFilePath: path + lockExt, + lockFile: nil, + } +} diff --git a/library/filelocker/file_locker_test.go b/library/filelocker/file_locker_test.go new file mode 100644 index 0000000..7283b6a --- /dev/null +++ b/library/filelocker/file_locker_test.go @@ -0,0 +1,25 @@ +package filelocker + +import ( + "fmt" + "log" + "testing" + "time" +) + +func TestFlocker(t *testing.T) { + // lock file first time - success + locker := NewFileLocker("D:\\smb\\feny\\goprojects\\dev\\aliyunpan") + e := LockFile(locker, 0755, true, 5*time.Second) + fmt.Println(e) + + // lock file again - fail + //time.Sleep(5 * time.Second) + //e = flock(locker, 0755, true, 5*time.Second) + //fmt.Println(e) + + // Unlock the file. + if err := UnlockFile(locker); err != nil { + log.Printf("funlock error: %s", err) + } +} diff --git a/library/filelocker/locker_solaris.go b/library/filelocker/locker_solaris.go new file mode 100644 index 0000000..d1ba095 --- /dev/null +++ b/library/filelocker/locker_solaris.go @@ -0,0 +1,57 @@ +package filelocker + +import ( + "os" + "syscall" + "time" +) + +// LockFile acquires an advisory lock on a file descriptor. +func LockFile(locker *FileLocker, mode os.FileMode, exclusive bool, timeout time.Duration) error { + f, err := os.OpenFile(locker.LockFilePath, os.O_CREATE, mode) + if err != nil { + return err + } + locker.lockFile = f + + var t time.Time + for { + // If we're beyond our timeout then return an error. + // This can only occur after we've attempted a flock once. + if t.IsZero() { + t = time.Now() + } else if timeout > 0 && time.Since(t) > timeout { + return ErrTimeout + } + var lock syscall.Flock_t + lock.Start = 0 + lock.Len = 0 + lock.Pid = 0 + lock.Whence = 0 + lock.Pid = 0 + if exclusive { + lock.Type = syscall.F_WRLCK + } else { + lock.Type = syscall.F_RDLCK + } + err := syscall.FcntlFlock(locker.lockFile.Fd(), syscall.F_SETLK, &lock) + if err == nil { + return nil + } else if err != syscall.EAGAIN { + return err + } + + // Wait for a bit and try again. + time.Sleep(50 * time.Millisecond) + } +} + +// UnlockFile releases an advisory lock on a file descriptor. +func UnlockFile(locker *FileLocker) error { + var lock syscall.Flock_t + lock.Start = 0 + lock.Len = 0 + lock.Type = syscall.F_UNLCK + lock.Whence = 0 + return syscall.FcntlFlock(uintptr(locker.lockFile.Fd()), syscall.F_SETLK, &lock) +} diff --git a/library/filelocker/locker_unix.go b/library/filelocker/locker_unix.go new file mode 100644 index 0000000..5bdbd4a --- /dev/null +++ b/library/filelocker/locker_unix.go @@ -0,0 +1,50 @@ +//go:build !windows && !plan9 && !solaris +// +build !windows,!plan9,!solaris + +package filelocker + +import ( + "os" + "syscall" + "time" +) + +// LockFile acquires an advisory lock on a file descriptor. +func LockFile(locker *FileLocker, mode os.FileMode, exclusive bool, timeout time.Duration) error { + f, err := os.OpenFile(locker.LockFilePath, os.O_CREATE, mode) + if err != nil { + return err + } + locker.lockFile = f + + var t time.Time + for { + // If we're beyond our timeout then return an error. + // This can only occur after we've attempted a flock once. + if t.IsZero() { + t = time.Now() + } else if timeout > 0 && time.Since(t) > timeout { + return ErrTimeout + } + flag := syscall.LOCK_SH + if exclusive { + flag = syscall.LOCK_EX + } + + // Otherwise attempt to obtain an exclusive lock. + err := syscall.Flock(int(locker.lockFile.Fd()), flag|syscall.LOCK_NB) + if err == nil { + return nil + } else if err != syscall.EWOULDBLOCK { + return err + } + + // Wait for a bit and try again. + time.Sleep(50 * time.Millisecond) + } +} + +// UnlockFile releases an advisory lock on a file descriptor. +func UnlockFile(locker *FileLocker) error { + return syscall.Flock(int(locker.lockFile.Fd()), syscall.LOCK_UN) +} diff --git a/library/filelocker/locker_windows.go b/library/filelocker/locker_windows.go new file mode 100644 index 0000000..71b06cb --- /dev/null +++ b/library/filelocker/locker_windows.go @@ -0,0 +1,86 @@ +package filelocker + +import ( + "os" + "syscall" + "time" + "unsafe" +) + +// LockFileEx code derived from golang build filemutex_windows.go @ v1.5.1 +var ( + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + procLockFileEx = modkernel32.NewProc("LockFileEx") + procUnlockFileEx = modkernel32.NewProc("UnlockFileEx") +) + +const ( + // see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx + flagLockExclusive = 2 + flagLockFailImmediately = 1 + + // see https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx + errLockViolation syscall.Errno = 0x21 +) + +func lockFileEx(h syscall.Handle, flags, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) { + r, _, err := procLockFileEx.Call(uintptr(h), uintptr(flags), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol))) + if r == 0 { + return err + } + return nil +} + +func unlockFileEx(h syscall.Handle, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) { + r, _, err := procUnlockFileEx.Call(uintptr(h), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)), 0) + if r == 0 { + return err + } + return nil +} + +// LockFile acquires an advisory lock on a file descriptor. +func LockFile(locker *FileLocker, mode os.FileMode, exclusive bool, timeout time.Duration) error { + // Create a separate lock file on windows because a process + // cannot share an exclusive lock on the same file. This is + // needed during Tx.WriteTo(). + f, err := os.OpenFile(locker.LockFilePath, os.O_CREATE, mode) + if err != nil { + return err + } + locker.lockFile = f + + var t time.Time + for { + // If we're beyond our timeout then return an error. + // This can only occur after we've attempted a flock once. + if t.IsZero() { + t = time.Now() + } else if timeout > 0 && time.Since(t) > timeout { + return ErrTimeout + } + + var flag uint32 = flagLockFailImmediately + if exclusive { + flag |= flagLockExclusive + } + + err := lockFileEx(syscall.Handle(locker.lockFile.Fd()), flag, 0, 1, 0, &syscall.Overlapped{}) + if err == nil { + return nil + } else if err != errLockViolation { + return err + } + + // Wait for a bit and try again. + time.Sleep(50 * time.Millisecond) + } +} + +// UnlockFile releases an advisory lock on a file descriptor. +func UnlockFile(locker *FileLocker) error { + err := unlockFileEx(syscall.Handle(locker.lockFile.Fd()), 0, 1, 0, &syscall.Overlapped{}) + locker.lockFile.Close() + os.Remove(locker.LockFilePath) + return err +}