Clone/aliyunpan
1
0
Fork 0
mirror of https://github.com/tickstep/aliyunpan.git synced 2025-01-23 14:32:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add webdav file list function

Browse Source
This commit is contained in:
feny 2021-12-30 20:24:33 +08:00
parent 43686488a6
commit 5a88effab9
5 changed files with 429 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/config/cache.go
View File

@ -46,7 +46,7 @@ func (pu *PanUser) CacheFilesDirectoriesList(pathStr string) (fdl aliyunpan.File
for _, f := range fdl {
f.Path = path.Join(pathStr, f.FileName)
}
return expires.NewDataExpires(fdl, 1*time.Minute)
return expires.NewDataExpires(fdl, 10*time.Minute)
})
if apiError != nil {
return

83
internal/webdav/dir.go
View File

@ -1,83 +0,0 @@
package webdav
import (
"context"
"mime"
"os"
"path"
"golang.org/x/net/webdav"
)
// NoSniffFileInfo wraps any generic FileInfo interface and bypasses mime type sniffing.
type NoSniffFileInfo struct {
os.FileInfo
}
func (w NoSniffFileInfo) ContentType(ctx context.Context) (contentType string, err error) {
if mimeType := mime.TypeByExtension(path.Ext(w.FileInfo.Name())); mimeType != "" {
// We can figure out the mime from the extension.
return mimeType, nil
} else {
// We can't figure out the mime type without sniffing, call it an octet stream.
return "application/octet-stream", nil
}
}
type WebDavDir struct {
webdav.Dir
NoSniff bool
}
func (d WebDavDir) Stat(ctx context.Context, name string) (os.FileInfo, error) {
// Skip wrapping if NoSniff is off
if !d.NoSniff {
return d.Dir.Stat(ctx, name)
}
info, err := d.Dir.Stat(ctx, name)
if err != nil {
return nil, err
}
return NoSniffFileInfo{info}, nil
}
func (d WebDavDir) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
// Skip wrapping if NoSniff is off
if !d.NoSniff {
return d.Dir.OpenFile(ctx, name, flag, perm)
}
file, err := d.Dir.OpenFile(ctx, name, flag, perm)
if err != nil {
return nil, err
}
return WebDavFile{File: file}, nil
}
type WebDavFile struct {
webdav.File
}
func (f WebDavFile) Stat() (os.FileInfo, error) {
info, err := f.File.Stat()
if err != nil {
return nil, err
}
return NoSniffFileInfo{info}, nil
}
func (f WebDavFile) Readdir(count int) (fis []os.FileInfo, err error) {
fis, err = f.File.Readdir(count)
if err != nil {
return nil, err
}
for i := range fis {
fis[i] = NoSniffFileInfo{fis[i]}
}
return fis, nil
}

149
internal/webdav/pan_client_proxy.go Normal file
View File

@ -0,0 +1,149 @@
package webdav
import (
"fmt"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan-api/aliyunpan/apierror"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/library-go/expires"
"github.com/tickstep/library-go/expires/cachemap"
"os"
"path"
"strings"
"time"
)
type PanClientProxy struct {
PanUser *config.PanUser
PanDriveId string
filePathCacheMap cachemap.CacheOpMap
fileDirectoryListCacheMap cachemap.CacheOpMap
}
// CACHE_EXPIRED_MINUTE 缓存过期分钟
const CACHE_EXPIRED_MINUTE = 60
// DeleteCache 删除含有 dirs 的缓存
func (p *PanClientProxy) deleteFilesDirectoriesListCache(dirs []string) {
cache := p.fileDirectoryListCacheMap.LazyInitCachePoolOp(p.PanDriveId)
for _, v := range dirs {
key := strings.TrimSuffix(v, "/")
_, ok := cache.Load(key)
if ok {
cache.Delete(key)
}
}
}
// DeleteOneCache 删除缓存
func (p *PanClientProxy) deleteOneFilesDirectoriesListCache(dirPath string) {
dirPath = strings.TrimSuffix(dirPath, "/")
ps := []string{dirPath}
p.deleteFilesDirectoriesListCache(ps)
}
// cacheFilesDirectoriesList 缓存文件夹下面的所有文件列表
func (p *PanClientProxy) cacheFilesDirectoriesList(pathStr string) (fdl aliyunpan.FileList, apiError *apierror.ApiError) {
pathStr = strings.TrimSuffix(pathStr, "/")
data := p.fileDirectoryListCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires {
fi, er := p.cacheFilePath(pathStr)
if er != nil {
return nil
}
fileListParam := &aliyunpan.FileListParam{
DriveId: p.PanDriveId,
ParentFileId: fi.FileId,
Limit: 200,
}
fdl, apiError = p.PanUser.PanClient().FileListGetAll(fileListParam)
if apiError != nil {
return nil
}
if len(fdl) == 0{
// 空目录不缓存
return nil
}
// construct full path
for _, f := range fdl {
f.Path = path.Join(pathStr, f.FileName)
}
p.cacheFilePathEntityList(fdl)
return expires.NewDataExpires(fdl, CACHE_EXPIRED_MINUTE*time.Minute)
})
if apiError != nil {
return
}
if data == nil {
return aliyunpan.FileList{}, nil
}
return data.Data().(aliyunpan.FileList), nil
}
// cacheFilePath 缓存文件绝对路径到网盘文件信息
func (p *PanClientProxy) cacheFilePath(pathStr string) (fe *aliyunpan.FileEntity, apiError *apierror.ApiError) {
pathStr = strings.TrimSuffix(pathStr, "/")
data := p.filePathCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires {
var fi *aliyunpan.FileEntity
fi, apiError = p.PanUser.PanClient().FileInfoByPath(p.PanDriveId, pathStr)
if apiError != nil {
return nil
}
return expires.NewDataExpires(fi, CACHE_EXPIRED_MINUTE*time.Minute)
})
if apiError != nil {
return
}
if data == nil {
return nil, nil
}
return data.Data().(*aliyunpan.FileEntity), nil
}
func (p *PanClientProxy) cacheFilePathEntity(fe *aliyunpan.FileEntity) {
pathStr := strings.TrimSuffix(fe.Path, "/")
p.filePathCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires {
return expires.NewDataExpires(fe, CACHE_EXPIRED_MINUTE*time.Minute)
})
}
func (p *PanClientProxy) cacheFilePathEntityList(fdl aliyunpan.FileList) {
for _,entity := range fdl {
pathStr := strings.TrimSuffix(entity.Path, "/")
p.filePathCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires {
return expires.NewDataExpires(entity, CACHE_EXPIRED_MINUTE*time.Minute)
})
}
}
func (p *PanClientProxy) FileInfoByPath(pathStr string) (fileInfo *aliyunpan.FileEntity, error *apierror.ApiError) {
return p.cacheFilePath(pathStr)
}
func (p *PanClientProxy) FileListGetAll(pathStr string) (aliyunpan.FileList, *apierror.ApiError) {
return p.cacheFilesDirectoriesList(pathStr)
}
func (p *PanClientProxy) Mkdir(pathStr string, perm os.FileMode) error {
if pathStr == "" {
return fmt.Errorf("unknown error")
}
pathStr = strings.ReplaceAll(pathStr, "\\", "/")
r,er := p.PanUser.PanClient().MkdirByFullPath(p.PanDriveId, pathStr)
if er != nil {
return er
}
// invalidate cache
p.deleteOneFilesDirectoriesListCache(path.Dir(pathStr))
if r.FileId != "" {
fe,_ := p.PanUser.PanClient().FileInfoById(p.PanDriveId, r.FileId)
if fe != nil {
fe.Path = pathStr
p.cacheFilePathEntity(fe)
}
return nil
}
return fmt.Errorf("unknown error")
}

18
internal/webdav/webdav_config.go
View File

@ -1,12 +1,14 @@
package webdav
import (
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/library-go/logger"
"golang.org/x/net/webdav"
"log"
"net"
"net/http"
"strconv"
"strings"
)
type WebdavUser struct {
@ -18,6 +20,8 @@ type WebdavUser struct {
type WebdavConfig struct {
// 指定Webdav使用哪个账号的云盘资源
PanUserId string `json:"panUserId"`
PanDriveId string `json:"panDriveId"`
PanUser *config.PanUser `json:"-"`
Address string `json:"address"`
Port int `json:"port"`
@ -28,6 +32,15 @@ type WebdavConfig struct {
func (w *WebdavConfig) StartServer() {
users := map[string]*User{}
for _,u := range w.Users {
fileItem,e := w.PanUser.PanClient().FileInfoByPath(w.PanDriveId, u.Scope)
if e != nil {
logger.Verboseln("scope not existed, shutting server")
return
}
wdfi := NewWebDavFileInfo(fileItem)
if wdfi.fullPath != "/" && strings.Index(wdfi.fullPath, "/") != 0 {
wdfi.fullPath = "/" + wdfi.fullPath
}
users[u.Username] = &User{
Username: u.Username,
Password: u.Password,
@ -39,6 +52,11 @@ func (w *WebdavConfig) StartServer() {
FileSystem: WebDavDir{
Dir: webdav.Dir(u.Scope),
NoSniff: false,
panClientProxy: &PanClientProxy{
PanUser: w.PanUser,
PanDriveId: w.PanDriveId,
},
fileInfo: wdfi,
},
LockSystem: webdav.NewMemLS(),
},

261
internal/webdav/webdav_file.go Normal file
View File

@ -0,0 +1,261 @@
package webdav
import (
"context"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/library-go/logger"
"mime"
"os"
"path"
"path/filepath"
"strings"
"time"
"golang.org/x/net/webdav"
)
// NoSniffFileInfo wraps any generic FileInfo interface and bypasses mime type sniffing.
type NoSniffFileInfo struct {
os.FileInfo
}
func (w NoSniffFileInfo) ContentType(ctx context.Context) (contentType string, err error) {
if mimeType := mime.TypeByExtension(path.Ext(w.FileInfo.Name())); mimeType != "" {
// We can figure out the mime from the extension.
return mimeType, nil
} else {
// We can't figure out the mime type without sniffing, call it an octet stream.
return "application/octet-stream", nil
}
}
// 文件系统
type WebDavDir struct {
webdav.Dir
NoSniff bool
panClientProxy *PanClientProxy
fileInfo WebDavFileInfo
}
// slashClean is equivalent to but slightly more efficient than
// path.Clean("/" + name).
func slashClean(name string) string {
if name == "" || name[0] != '/' {
name = "/" + name
}
return path.Clean(name)
}
// formatAbsoluteName 将name名称更改为绝对路径
func (d WebDavDir) formatAbsoluteName(pathStr string) string {
if strings.Index(pathStr, "/") != 0 {
pathStr = d.fileInfo.fullPath + "/" + pathStr
}
return pathStr
}
func (d WebDavDir) resolve(name string) string {
// This implementation is based on Dir.Open's code in the standard net/http package.
if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 ||
strings.Contains(name, "\x00") {
return ""
}
dir := string(d.Dir)
if dir == "" {
dir = "."
}
return filepath.Join(dir, filepath.FromSlash(slashClean(name)))
}
func (d WebDavDir) Mkdir(ctx context.Context, name string, perm os.FileMode) error {
if name = d.resolve(name); name == "" {
return os.ErrNotExist
}
return d.panClientProxy.Mkdir(name, perm)
}
func (d WebDavDir) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
if name == "" {
return WebDavFile{
panClientProxy: d.panClientProxy,
nameSnapshot: d.fileInfo,
childrenSnapshot: nil,
listPos: 0,
readPos: 0,
writePos: 0,
}, nil
}
fileItem,e := d.panClientProxy.FileInfoByPath(d.formatAbsoluteName(name))
if e != nil {
logger.Verboseln("OpenFile failed, file path not existed: " + d.formatAbsoluteName(name))
return nil, e
}
wdfi := NewWebDavFileInfo(fileItem)
wdfi.fullPath = d.formatAbsoluteName(name)
return WebDavFile{
panClientProxy: d.panClientProxy,
nameSnapshot: wdfi,
childrenSnapshot: nil,
listPos: 0,
readPos: 0,
writePos: 0,
}, nil
}
func (d WebDavDir) RemoveAll(ctx context.Context, name string) error {
if name = d.resolve(name); name == "" {
return os.ErrNotExist
}
if name == filepath.Clean(string(d.Dir)) {
// Prohibit removing the virtual root directory.
return os.ErrInvalid
}
return os.RemoveAll(name)
}
func (d WebDavDir) Rename(ctx context.Context, oldName, newName string) error {
if oldName = d.resolve(oldName); oldName == "" {
return os.ErrNotExist
}
if newName = d.resolve(newName); newName == "" {
return os.ErrNotExist
}
if root := filepath.Clean(string(d.Dir)); root == oldName || root == newName {
// Prohibit renaming from or to the virtual root directory.
return os.ErrInvalid
}
return os.Rename(oldName, newName)
}
func (d WebDavDir) Stat(ctx context.Context, name string) (os.FileInfo, error) {
f := &d.fileInfo
if name != "" {
fileItem,e := d.panClientProxy.FileInfoByPath(d.formatAbsoluteName(name))
if e != nil {
logger.Verboseln("file path not existed: " + d.formatAbsoluteName(name))
return nil, e
}
*f = NewWebDavFileInfo(fileItem)
}
return f, nil
}
// WebDavFile 文件实例
type WebDavFile struct {
webdav.File
panClientProxy *PanClientProxy
nameSnapshot WebDavFileInfo
childrenSnapshot []WebDavFileInfo
listPos int
readPos int64
writePos int64
}
func (f WebDavFile) Close() error {
return nil
}
func (f WebDavFile) Read(p []byte) (int, error) {
return 0, nil
}
// Readdir 获取文件目录
func (f WebDavFile) Readdir(count int) (fis []os.FileInfo, err error) {
if f.childrenSnapshot == nil || len(f.childrenSnapshot) == 0 {
fileList, e := f.panClientProxy.FileListGetAll(f.nameSnapshot.fullPath)
if e != nil {
return nil, e
}
for _,fileItem := range fileList {
wdfi := NewWebDavFileInfo(fileItem)
wdfi.fullPath = f.nameSnapshot.fullPath + "/" + wdfi.name
f.childrenSnapshot = append(f.childrenSnapshot, wdfi)
}
}
realCount := count
if (f.listPos + realCount) > len(f.childrenSnapshot) {
realCount = len(f.childrenSnapshot) - f.listPos
}
if realCount == 0 {
realCount = len(f.childrenSnapshot)
}
fis = make([]os.FileInfo, realCount)
idx := 0
for idx < realCount {
fis[idx] = &f.childrenSnapshot[f.listPos + idx]
idx ++
}
return fis, nil
}
func (f WebDavFile) Seek(off int64, whence int) (int64, error) {
f.readPos += off
return f.readPos, nil
}
func (f WebDavFile) Stat() (os.FileInfo, error) {
return &f.nameSnapshot, nil
}
func (f WebDavFile) Write(p []byte) (int, error) {
return 0, nil
}
// WebDavFileInfo 文件信息
type WebDavFileInfo struct {
os.FileInfo
fileId string
name string
size int64
mode os.FileMode
modTime time.Time
fullPath string
}
func NewWebDavFileInfo(fileItem *aliyunpan.FileEntity) WebDavFileInfo {
var LOC, _ = time.LoadLocation("Asia/Shanghai")
t,_ := time.ParseInLocation("2006-01-02 15:04:05", fileItem.UpdatedAt, LOC)
fm := os.FileMode(0)
if fileItem.IsFolder() {
fm = os.ModeDir
}
return WebDavFileInfo{
fileId: fileItem.FileId,
name: fileItem.FileName,
size: fileItem.FileSize,
mode: fm,
modTime: t,
fullPath: fileItem.Path,
}
}
func (f *WebDavFileInfo) Name() string { return f.name }
func (f *WebDavFileInfo) Size() int64 { return f.size }
func (f *WebDavFileInfo) Mode() os.FileMode { return f.mode }
func (f *WebDavFileInfo) ModTime() time.Time { return f.modTime }
func (f *WebDavFileInfo) IsDir() bool { return f.mode.IsDir() }
func (f *WebDavFileInfo) Sys() interface{} { return nil }
func (f *WebDavFileInfo) ContentType(ctx context.Context) (contentType string, err error) {
if mimeType := mime.TypeByExtension(path.Ext(f.Name())); mimeType != "" {
// We can figure out the mime from the extension.
return mimeType, nil
} else {
// We can't figure out the mime type without sniffing, call it an octet stream.
return "application/octet-stream", nil
}
}