From 5a88effab99b81de263d8401bf5f0b5afb134a04 Mon Sep 17 00:00:00 2001 From: feny Date: Thu, 30 Dec 2021 20:24:33 +0800 Subject: [PATCH] add webdav file list function --- internal/config/cache.go | 2 +- internal/webdav/dir.go | 83 --------- internal/webdav/pan_client_proxy.go | 149 ++++++++++++++++ internal/webdav/webdav_config.go | 18 ++ internal/webdav/webdav_file.go | 261 ++++++++++++++++++++++++++++ 5 files changed, 429 insertions(+), 84 deletions(-) delete mode 100644 internal/webdav/dir.go create mode 100644 internal/webdav/pan_client_proxy.go create mode 100644 internal/webdav/webdav_file.go diff --git a/internal/config/cache.go b/internal/config/cache.go index 40442b6..4bc6515 100644 --- a/internal/config/cache.go +++ b/internal/config/cache.go @@ -46,7 +46,7 @@ func (pu *PanUser) CacheFilesDirectoriesList(pathStr string) (fdl aliyunpan.File for _, f := range fdl { f.Path = path.Join(pathStr, f.FileName) } - return expires.NewDataExpires(fdl, 1*time.Minute) + return expires.NewDataExpires(fdl, 10*time.Minute) }) if apiError != nil { return diff --git a/internal/webdav/dir.go b/internal/webdav/dir.go deleted file mode 100644 index 9593ec7..0000000 --- a/internal/webdav/dir.go +++ /dev/null @@ -1,83 +0,0 @@ -package webdav - -import ( - "context" - "mime" - "os" - "path" - - "golang.org/x/net/webdav" -) - -// NoSniffFileInfo wraps any generic FileInfo interface and bypasses mime type sniffing. -type NoSniffFileInfo struct { - os.FileInfo -} - -func (w NoSniffFileInfo) ContentType(ctx context.Context) (contentType string, err error) { - if mimeType := mime.TypeByExtension(path.Ext(w.FileInfo.Name())); mimeType != "" { - // We can figure out the mime from the extension. - return mimeType, nil - } else { - // We can't figure out the mime type without sniffing, call it an octet stream. - return "application/octet-stream", nil - } -} - -type WebDavDir struct { - webdav.Dir - NoSniff bool -} - -func (d WebDavDir) Stat(ctx context.Context, name string) (os.FileInfo, error) { - // Skip wrapping if NoSniff is off - if !d.NoSniff { - return d.Dir.Stat(ctx, name) - } - - info, err := d.Dir.Stat(ctx, name) - if err != nil { - return nil, err - } - - return NoSniffFileInfo{info}, nil -} - -func (d WebDavDir) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) { - // Skip wrapping if NoSniff is off - if !d.NoSniff { - return d.Dir.OpenFile(ctx, name, flag, perm) - } - - file, err := d.Dir.OpenFile(ctx, name, flag, perm) - if err != nil { - return nil, err - } - - return WebDavFile{File: file}, nil -} - -type WebDavFile struct { - webdav.File -} - -func (f WebDavFile) Stat() (os.FileInfo, error) { - info, err := f.File.Stat() - if err != nil { - return nil, err - } - - return NoSniffFileInfo{info}, nil -} - -func (f WebDavFile) Readdir(count int) (fis []os.FileInfo, err error) { - fis, err = f.File.Readdir(count) - if err != nil { - return nil, err - } - - for i := range fis { - fis[i] = NoSniffFileInfo{fis[i]} - } - return fis, nil -} diff --git a/internal/webdav/pan_client_proxy.go b/internal/webdav/pan_client_proxy.go new file mode 100644 index 0000000..d1f1e20 --- /dev/null +++ b/internal/webdav/pan_client_proxy.go @@ -0,0 +1,149 @@ +package webdav + +import ( + "fmt" + "github.com/tickstep/aliyunpan-api/aliyunpan" + "github.com/tickstep/aliyunpan-api/aliyunpan/apierror" + "github.com/tickstep/aliyunpan/internal/config" + "github.com/tickstep/library-go/expires" + "github.com/tickstep/library-go/expires/cachemap" + "os" + "path" + "strings" + "time" +) + +type PanClientProxy struct { + PanUser *config.PanUser + PanDriveId string + + filePathCacheMap cachemap.CacheOpMap + fileDirectoryListCacheMap cachemap.CacheOpMap +} + +// CACHE_EXPIRED_MINUTE 缓存过期分钟 +const CACHE_EXPIRED_MINUTE = 60 + +// DeleteCache 删除含有 dirs 的缓存 +func (p *PanClientProxy) deleteFilesDirectoriesListCache(dirs []string) { + cache := p.fileDirectoryListCacheMap.LazyInitCachePoolOp(p.PanDriveId) + for _, v := range dirs { + key := strings.TrimSuffix(v, "/") + _, ok := cache.Load(key) + if ok { + cache.Delete(key) + } + } +} + +// DeleteOneCache 删除缓存 +func (p *PanClientProxy) deleteOneFilesDirectoriesListCache(dirPath string) { + dirPath = strings.TrimSuffix(dirPath, "/") + ps := []string{dirPath} + p.deleteFilesDirectoriesListCache(ps) +} + +// cacheFilesDirectoriesList 缓存文件夹下面的所有文件列表 +func (p *PanClientProxy) cacheFilesDirectoriesList(pathStr string) (fdl aliyunpan.FileList, apiError *apierror.ApiError) { + pathStr = strings.TrimSuffix(pathStr, "/") + data := p.fileDirectoryListCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires { + fi, er := p.cacheFilePath(pathStr) + if er != nil { + return nil + } + fileListParam := &aliyunpan.FileListParam{ + DriveId: p.PanDriveId, + ParentFileId: fi.FileId, + Limit: 200, + } + fdl, apiError = p.PanUser.PanClient().FileListGetAll(fileListParam) + if apiError != nil { + return nil + } + if len(fdl) == 0{ + // 空目录不缓存 + return nil + } + // construct full path + for _, f := range fdl { + f.Path = path.Join(pathStr, f.FileName) + } + p.cacheFilePathEntityList(fdl) + return expires.NewDataExpires(fdl, CACHE_EXPIRED_MINUTE*time.Minute) + }) + if apiError != nil { + return + } + if data == nil { + return aliyunpan.FileList{}, nil + } + return data.Data().(aliyunpan.FileList), nil +} + +// cacheFilePath 缓存文件绝对路径到网盘文件信息 +func (p *PanClientProxy) cacheFilePath(pathStr string) (fe *aliyunpan.FileEntity, apiError *apierror.ApiError) { + pathStr = strings.TrimSuffix(pathStr, "/") + data := p.filePathCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires { + var fi *aliyunpan.FileEntity + fi, apiError = p.PanUser.PanClient().FileInfoByPath(p.PanDriveId, pathStr) + if apiError != nil { + return nil + } + return expires.NewDataExpires(fi, CACHE_EXPIRED_MINUTE*time.Minute) + }) + if apiError != nil { + return + } + if data == nil { + return nil, nil + } + return data.Data().(*aliyunpan.FileEntity), nil +} + +func (p *PanClientProxy) cacheFilePathEntity(fe *aliyunpan.FileEntity) { + pathStr := strings.TrimSuffix(fe.Path, "/") + p.filePathCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires { + return expires.NewDataExpires(fe, CACHE_EXPIRED_MINUTE*time.Minute) + }) +} + +func (p *PanClientProxy) cacheFilePathEntityList(fdl aliyunpan.FileList) { + for _,entity := range fdl { + pathStr := strings.TrimSuffix(entity.Path, "/") + p.filePathCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires { + return expires.NewDataExpires(entity, CACHE_EXPIRED_MINUTE*time.Minute) + }) + } +} + + +func (p *PanClientProxy) FileInfoByPath(pathStr string) (fileInfo *aliyunpan.FileEntity, error *apierror.ApiError) { + return p.cacheFilePath(pathStr) +} + +func (p *PanClientProxy) FileListGetAll(pathStr string) (aliyunpan.FileList, *apierror.ApiError) { + return p.cacheFilesDirectoriesList(pathStr) +} + +func (p *PanClientProxy) Mkdir(pathStr string, perm os.FileMode) error { + if pathStr == "" { + return fmt.Errorf("unknown error") + } + pathStr = strings.ReplaceAll(pathStr, "\\", "/") + r,er := p.PanUser.PanClient().MkdirByFullPath(p.PanDriveId, pathStr) + if er != nil { + return er + } + // invalidate cache + p.deleteOneFilesDirectoriesListCache(path.Dir(pathStr)) + + if r.FileId != "" { + fe,_ := p.PanUser.PanClient().FileInfoById(p.PanDriveId, r.FileId) + if fe != nil { + fe.Path = pathStr + p.cacheFilePathEntity(fe) + } + return nil + } + return fmt.Errorf("unknown error") +} \ No newline at end of file diff --git a/internal/webdav/webdav_config.go b/internal/webdav/webdav_config.go index 771ae7c..ebde6b3 100644 --- a/internal/webdav/webdav_config.go +++ b/internal/webdav/webdav_config.go @@ -1,12 +1,14 @@ package webdav import ( + "github.com/tickstep/aliyunpan/internal/config" "github.com/tickstep/library-go/logger" "golang.org/x/net/webdav" "log" "net" "net/http" "strconv" + "strings" ) type WebdavUser struct { @@ -18,6 +20,8 @@ type WebdavUser struct { type WebdavConfig struct { // 指定Webdav使用哪个账号的云盘资源 PanUserId string `json:"panUserId"` + PanDriveId string `json:"panDriveId"` + PanUser *config.PanUser `json:"-"` Address string `json:"address"` Port int `json:"port"` @@ -28,6 +32,15 @@ type WebdavConfig struct { func (w *WebdavConfig) StartServer() { users := map[string]*User{} for _,u := range w.Users { + fileItem,e := w.PanUser.PanClient().FileInfoByPath(w.PanDriveId, u.Scope) + if e != nil { + logger.Verboseln("scope not existed, shutting server") + return + } + wdfi := NewWebDavFileInfo(fileItem) + if wdfi.fullPath != "/" && strings.Index(wdfi.fullPath, "/") != 0 { + wdfi.fullPath = "/" + wdfi.fullPath + } users[u.Username] = &User{ Username: u.Username, Password: u.Password, @@ -39,6 +52,11 @@ func (w *WebdavConfig) StartServer() { FileSystem: WebDavDir{ Dir: webdav.Dir(u.Scope), NoSniff: false, + panClientProxy: &PanClientProxy{ + PanUser: w.PanUser, + PanDriveId: w.PanDriveId, + }, + fileInfo: wdfi, }, LockSystem: webdav.NewMemLS(), }, diff --git a/internal/webdav/webdav_file.go b/internal/webdav/webdav_file.go new file mode 100644 index 0000000..584e5c9 --- /dev/null +++ b/internal/webdav/webdav_file.go @@ -0,0 +1,261 @@ +package webdav + +import ( + "context" + "github.com/tickstep/aliyunpan-api/aliyunpan" + "github.com/tickstep/library-go/logger" + "mime" + "os" + "path" + "path/filepath" + "strings" + "time" + + "golang.org/x/net/webdav" +) + +// NoSniffFileInfo wraps any generic FileInfo interface and bypasses mime type sniffing. +type NoSniffFileInfo struct { + os.FileInfo +} + +func (w NoSniffFileInfo) ContentType(ctx context.Context) (contentType string, err error) { + if mimeType := mime.TypeByExtension(path.Ext(w.FileInfo.Name())); mimeType != "" { + // We can figure out the mime from the extension. + return mimeType, nil + } else { + // We can't figure out the mime type without sniffing, call it an octet stream. + return "application/octet-stream", nil + } +} + + +// 文件系统 +type WebDavDir struct { + webdav.Dir + NoSniff bool + panClientProxy *PanClientProxy + fileInfo WebDavFileInfo +} + +// slashClean is equivalent to but slightly more efficient than +// path.Clean("/" + name). +func slashClean(name string) string { + if name == "" || name[0] != '/' { + name = "/" + name + } + return path.Clean(name) +} + +// formatAbsoluteName 将name名称更改为绝对路径 +func (d WebDavDir) formatAbsoluteName(pathStr string) string { + if strings.Index(pathStr, "/") != 0 { + pathStr = d.fileInfo.fullPath + "/" + pathStr + } + return pathStr +} + +func (d WebDavDir) resolve(name string) string { + // This implementation is based on Dir.Open's code in the standard net/http package. + if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || + strings.Contains(name, "\x00") { + return "" + } + dir := string(d.Dir) + if dir == "" { + dir = "." + } + return filepath.Join(dir, filepath.FromSlash(slashClean(name))) +} + +func (d WebDavDir) Mkdir(ctx context.Context, name string, perm os.FileMode) error { + if name = d.resolve(name); name == "" { + return os.ErrNotExist + } + return d.panClientProxy.Mkdir(name, perm) +} + +func (d WebDavDir) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) { + if name == "" { + return WebDavFile{ + panClientProxy: d.panClientProxy, + nameSnapshot: d.fileInfo, + childrenSnapshot: nil, + listPos: 0, + readPos: 0, + writePos: 0, + }, nil + } + + fileItem,e := d.panClientProxy.FileInfoByPath(d.formatAbsoluteName(name)) + if e != nil { + logger.Verboseln("OpenFile failed, file path not existed: " + d.formatAbsoluteName(name)) + return nil, e + } + wdfi := NewWebDavFileInfo(fileItem) + wdfi.fullPath = d.formatAbsoluteName(name) + return WebDavFile{ + panClientProxy: d.panClientProxy, + nameSnapshot: wdfi, + childrenSnapshot: nil, + listPos: 0, + readPos: 0, + writePos: 0, + }, nil +} + +func (d WebDavDir) RemoveAll(ctx context.Context, name string) error { + if name = d.resolve(name); name == "" { + return os.ErrNotExist + } + if name == filepath.Clean(string(d.Dir)) { + // Prohibit removing the virtual root directory. + return os.ErrInvalid + } + return os.RemoveAll(name) +} + +func (d WebDavDir) Rename(ctx context.Context, oldName, newName string) error { + if oldName = d.resolve(oldName); oldName == "" { + return os.ErrNotExist + } + if newName = d.resolve(newName); newName == "" { + return os.ErrNotExist + } + if root := filepath.Clean(string(d.Dir)); root == oldName || root == newName { + // Prohibit renaming from or to the virtual root directory. + return os.ErrInvalid + } + return os.Rename(oldName, newName) +} + +func (d WebDavDir) Stat(ctx context.Context, name string) (os.FileInfo, error) { + f := &d.fileInfo + if name != "" { + fileItem,e := d.panClientProxy.FileInfoByPath(d.formatAbsoluteName(name)) + if e != nil { + logger.Verboseln("file path not existed: " + d.formatAbsoluteName(name)) + return nil, e + } + *f = NewWebDavFileInfo(fileItem) + } + return f, nil +} + + + + + +// WebDavFile 文件实例 +type WebDavFile struct { + webdav.File + + panClientProxy *PanClientProxy + nameSnapshot WebDavFileInfo + childrenSnapshot []WebDavFileInfo + + listPos int + readPos int64 + writePos int64 +} + +func (f WebDavFile) Close() error { + return nil +} + +func (f WebDavFile) Read(p []byte) (int, error) { + return 0, nil +} + +// Readdir 获取文件目录 +func (f WebDavFile) Readdir(count int) (fis []os.FileInfo, err error) { + if f.childrenSnapshot == nil || len(f.childrenSnapshot) == 0 { + fileList, e := f.panClientProxy.FileListGetAll(f.nameSnapshot.fullPath) + if e != nil { + return nil, e + } + for _,fileItem := range fileList { + wdfi := NewWebDavFileInfo(fileItem) + wdfi.fullPath = f.nameSnapshot.fullPath + "/" + wdfi.name + f.childrenSnapshot = append(f.childrenSnapshot, wdfi) + } + } + + realCount := count + if (f.listPos + realCount) > len(f.childrenSnapshot) { + realCount = len(f.childrenSnapshot) - f.listPos + } + if realCount == 0 { + realCount = len(f.childrenSnapshot) + } + + fis = make([]os.FileInfo, realCount) + idx := 0 + for idx < realCount { + fis[idx] = &f.childrenSnapshot[f.listPos + idx] + idx ++ + } + return fis, nil +} + +func (f WebDavFile) Seek(off int64, whence int) (int64, error) { + f.readPos += off + return f.readPos, nil +} + +func (f WebDavFile) Stat() (os.FileInfo, error) { + return &f.nameSnapshot, nil +} + +func (f WebDavFile) Write(p []byte) (int, error) { + return 0, nil +} + + + + + + +// WebDavFileInfo 文件信息 +type WebDavFileInfo struct { + os.FileInfo + fileId string + name string + size int64 + mode os.FileMode + modTime time.Time + fullPath string +} + +func NewWebDavFileInfo(fileItem *aliyunpan.FileEntity) WebDavFileInfo { + var LOC, _ = time.LoadLocation("Asia/Shanghai") + t,_ := time.ParseInLocation("2006-01-02 15:04:05", fileItem.UpdatedAt, LOC) + fm := os.FileMode(0) + if fileItem.IsFolder() { + fm = os.ModeDir + } + return WebDavFileInfo{ + fileId: fileItem.FileId, + name: fileItem.FileName, + size: fileItem.FileSize, + mode: fm, + modTime: t, + fullPath: fileItem.Path, + } +} + +func (f *WebDavFileInfo) Name() string { return f.name } +func (f *WebDavFileInfo) Size() int64 { return f.size } +func (f *WebDavFileInfo) Mode() os.FileMode { return f.mode } +func (f *WebDavFileInfo) ModTime() time.Time { return f.modTime } +func (f *WebDavFileInfo) IsDir() bool { return f.mode.IsDir() } +func (f *WebDavFileInfo) Sys() interface{} { return nil } +func (f *WebDavFileInfo) ContentType(ctx context.Context) (contentType string, err error) { + if mimeType := mime.TypeByExtension(path.Ext(f.Name())); mimeType != "" { + // We can figure out the mime from the extension. + return mimeType, nil + } else { + // We can't figure out the mime type without sniffing, call it an octet stream. + return "application/octet-stream", nil + } +}