diff --git a/drivers/all.go b/drivers/all.go index 15fbf2b96..7e1c24bba 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -17,6 +17,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_share" + _ "github.com/OpenListTeam/OpenList/v4/drivers/autoindex" _ "github.com/OpenListTeam/OpenList/v4/drivers/azure_blob" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_netdisk" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo" diff --git a/drivers/autoindex/driver.go b/drivers/autoindex/driver.go new file mode 100644 index 000000000..5758ec6cd --- /dev/null +++ b/drivers/autoindex/driver.go @@ -0,0 +1,169 @@ +package autoindex + +import ( + "context" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/antchfx/htmlquery" + "github.com/antchfx/xpath" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type AutoIndex struct { + model.Storage + Addition + itemXPath *xpath.Expr + nameXPath *xpath.Expr + modifiedXPath *xpath.Expr + sizeXPath *xpath.Expr + ignores map[string]any +} + +func (d *AutoIndex) Config() driver.Config { + return config +} + +func (d *AutoIndex) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *AutoIndex) Init(ctx context.Context) error { + var err error + d.itemXPath, err = xpath.Compile(d.ItemXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Item XPath") + } + d.nameXPath, err = xpath.Compile(d.NameXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Name XPath") + } + if len(d.ModifiedXPath) > 0 { + d.modifiedXPath, err = xpath.Compile(d.ModifiedXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Modified XPath") + } + } + if len(d.SizeXPath) > 0 { + d.sizeXPath, err = xpath.Compile(d.SizeXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Size XPath") + } + } + ignores := strings.Split(d.IgnoreFileNames, "\n") + d.ignores = make(map[string]any, len(ignores)) + for _, i := range ignores { + i = strings.TrimSpace(i) + if len(i) == 0 { + continue + } + d.ignores[i] = struct{}{} + } + hasScheme := strings.Contains(d.URL, "://") + hasSuffix := strings.HasSuffix(d.URL, "/") + if !hasScheme || !hasSuffix { + if !hasSuffix { + d.URL = d.URL + "/" + } + if !hasScheme { + d.URL = "https://" + d.URL + } + op.MustSaveDriverStorage(d) + } + return nil +} + +func (d *AutoIndex) Drop(ctx context.Context) error { + return nil +} + +func (d *AutoIndex) GetRoot(ctx context.Context) (model.Obj, error) { + return &model.Object{ + Name: op.RootName, + Path: d.URL, + Modified: d.Modified, + Mask: model.Locked, + IsFolder: true, + }, nil +} + +func (d *AutoIndex) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + res, err := base.RestyClient.R(). + SetContext(ctx). + SetDoNotParseResponse(true). + Get(dir.GetPath()) + if err != nil { + return nil, errors.WithMessagef(err, "failed to get url [%s]", dir.GetPath()) + } + defer res.RawResponse.Body.Close() + doc, err := htmlquery.Parse(res.RawBody()) + if err != nil { + return nil, errors.WithMessagef(err, "failed to parse [%s]", dir.GetPath()) + } + itemsIter := d.itemXPath.Select(htmlquery.CreateXPathNavigator(doc)) + var objs []model.Obj + for itemsIter.MoveNext() { + nameFull, err := parseString(d.nameXPath.Evaluate(itemsIter.Current().Copy())) + if err != nil { + log.Warnf("skip invalid name evaluating result: %v", err) + continue + } + nameFull = strings.TrimSpace(nameFull) + name, isDir := strings.CutSuffix(nameFull, "/") + if _, ok := d.ignores[name]; ok { + continue + } + var size int64 = 0 + exact := false + modified := time.Now() + if d.sizeXPath != nil { + size, exact, err = parseSize(d.sizeXPath.Evaluate(itemsIter.Current().Copy())) + if err != nil { + log.Errorf("failed to parse size of %s: %v", name, err) + } + } + if d.modifiedXPath != nil { + modified, err = parseTime(d.modifiedXPath.Evaluate(itemsIter.Current().Copy()), d.ModifiedTimeFormat) + if err != nil { + log.Errorf("failed to parse modified time of %s: %v", name, err) + } + } + var o model.Obj = &model.Object{ + Name: name, + IsFolder: isDir, + Path: dir.GetPath() + nameFull, + Modified: modified, + Size: size, + } + if exact { + o = &exactSizeObj{Obj: o} + } + objs = append(objs, o) + } + return objs, nil +} + +func (d *AutoIndex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if _, ok := file.(*exactSizeObj); ok || args.Redirect { + return &model.Link{URL: file.GetPath()}, nil + } + res, err := base.RestyClient.R(). + SetContext(ctx). + SetDoNotParseResponse(true). + Head(file.GetPath()) + if err != nil { + return nil, errors.WithMessagef(err, "failed to head [%s]", file.GetPath()) + } + _ = res.RawResponse.Body.Close() + return &model.Link{ + URL: file.GetPath(), + ContentLength: res.RawResponse.ContentLength, + }, nil +} + +var _ driver.Driver = (*AutoIndex)(nil) diff --git a/drivers/autoindex/meta.go b/drivers/autoindex/meta.go new file mode 100644 index 000000000..8ebae5924 --- /dev/null +++ b/drivers/autoindex/meta.go @@ -0,0 +1,29 @@ +package autoindex + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + URL string `json:"url" required:"true"` + ItemXPath string `json:"item_xpath" required:"true"` + NameXPath string `json:"name_xpath" required:"true"` + ModifiedXPath string `json:"modified_xpath"` + SizeXPath string `json:"size_xpath"` + IgnoreFileNames string `json:"ignore_file_names" type:"text" default:".\n..\nParent Directory\nUp"` + ModifiedTimeFormat string `json:"modified_time_format" default:"02-Jan-2006 15:04" help:"Must be based on the time point Mon Jan 2 15:04:05 -0700 MST 2006"` +} + +var config = driver.Config{ + Name: "AutoIndex", + LocalSort: true, + CheckStatus: true, + NoUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &AutoIndex{} + }) +} diff --git a/drivers/autoindex/types.go b/drivers/autoindex/types.go new file mode 100644 index 000000000..48ec396f5 --- /dev/null +++ b/drivers/autoindex/types.go @@ -0,0 +1,13 @@ +package autoindex + +import ( + "fmt" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +var ( + errEmptyEvaluateResult = fmt.Errorf("empty result") +) + +type exactSizeObj struct{ model.Obj } diff --git a/drivers/autoindex/util.go b/drivers/autoindex/util.go new file mode 100644 index 000000000..f5f36a736 --- /dev/null +++ b/drivers/autoindex/util.go @@ -0,0 +1,116 @@ +package autoindex + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/antchfx/xpath" + "github.com/pkg/errors" +) + +var units = map[string]int64{ + "": 1, + "b": 1, + "byte": 1, + "bytes": 1, + "k": 1 << 10, + "kb": 1 << 10, + "kib": 1 << 10, + "m": 1 << 20, + "mb": 1 << 20, + "mib": 1 << 20, + "g": 1 << 30, + "gb": 1 << 30, + "gib": 1 << 30, + "t": 1 << 40, + "tb": 1 << 40, + "tib": 1 << 40, + "p": 1 << 50, + "pb": 1 << 50, + "pib": 1 << 50, +} + +func splitUnit(s string) (string, string) { + for i := len(s) - 1; i >= 0; i-- { + if s[i] >= '0' && s[i] <= '9' { + return strings.TrimSpace(s[:i+1]), strings.TrimSpace(s[i+1:]) + } + } + return "", s +} + +func parseSize(a any) (int64, bool, error) { + // 第二个返回值exact表示大小是否精确 + if f, ok := a.(float64); ok { + return int64(f), false, nil + } + s, err := parseString(a) + if errors.Is(err, errEmptyEvaluateResult) { + // 可能是错误,也可能确实大小为0 + // 如果确实大小为0,大概率不会下载,exact返回false也不会有什么性能损失 + // 如果是错误,exact返回true会导致本地代理出错,综合来看返回false更好 + return 0, false, nil + } + if err != nil { + return 0, false, err + } + s = strings.TrimSpace(s) + if s == "-" { + return 0, false, nil + } + nbs, unit := splitUnit(s) + mul, ok := units[strings.ToLower(unit)] + exact := mul == 1 + if !ok { + mul = 1 + // 推测无单位,exact应为false + } + nb, err := strconv.ParseInt(nbs, 10, 64) + if err != nil { + fnb, err := strconv.ParseFloat(nbs, 64) + if err != nil { + return 0, false, fmt.Errorf("failed to convert %s to number", nbs) + } + nb = int64(fnb * float64(mul)) + exact = false + } else { + nb = nb * mul + } + return nb, exact, nil +} + +func parseString(res any) (string, error) { + if r, ok := res.(string); ok { + if len(r) == 0 { + return "", errEmptyEvaluateResult + } + return r, nil + } + n, ok := res.(*xpath.NodeIterator) + if !ok { + return "", fmt.Errorf("unsupported evaluating result") + } + if !n.MoveNext() { + return "", fmt.Errorf("no matched nodes") + } + ns := n.Current().Value() + if len(ns) == 0 { + return "", errEmptyEvaluateResult + } + return ns, nil +} + +func parseTime(res any, format string) (time.Time, error) { + s, err := parseString(res) + if err != nil { + return time.Now(), err + } + s = strings.TrimSpace(s) + t, err := time.Parse(format, s) + if err != nil { + return time.Now(), errors.WithMessagef(err, "failed to convert %s to time", s) + } + return t, nil +} diff --git a/drivers/autoindex/util_test.go b/drivers/autoindex/util_test.go new file mode 100644 index 000000000..ba743b943 --- /dev/null +++ b/drivers/autoindex/util_test.go @@ -0,0 +1,49 @@ +package autoindex + +import ( + "testing" +) + +type wantType struct { + v int64 + exact bool + error bool +} + +func TestParseSize(t *testing.T) { + tests := []struct { + input string + want wantType + }{ + {"100", wantType{100, true, false}}, + {"1k", wantType{1024, false, false}}, + {"1kb", wantType{1024, false, false}}, + {"1K", wantType{1024, false, false}}, // case insensitive + {"1.5m", wantType{1572864, false, false}}, // 1.5 * 1024^2 + {"500 bytes", wantType{500, true, false}}, + {"-", wantType{0, false, false}}, + {"", wantType{0, false, false}}, + {"abc", wantType{0, false, true}}, + {"1.5GB", wantType{1610612736, false, false}}, // 1.5 * 1024^3 + {"2t", wantType{2199023255552, false, false}}, // 2 * 1024^4 + {"1p", wantType{1125899906842624, false, false}}, // 1 * 1024^5 + {"0", wantType{0, true, false}}, + {" 100 ", wantType{100, true, false}}, // trimmed + {"100b", wantType{100, true, false}}, + {"1gib", wantType{1073741824, false, false}}, // 1024^3 + {"1z", wantType{1, false, false}}, // invalid unit, mul=1 + {"1.5", wantType{1, false, false}}, // float without unit, truncated + {"2.7k", wantType{2764, false, false}}, // 2.7 * 1024 truncated + {"1.0g", wantType{1073741824, false, false}}, // 1.0 * 1024^3 + {"invalid", wantType{0, false, true}}, + {"123xyz", wantType{123, false, false}}, // unit not found, mul=1 + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got, exact, err := parseSize(tt.input) + if got != tt.want.v || exact != tt.want.exact || (err != nil) != tt.want.error { + t.Errorf("ParseSize(%q) = (%d, %t, %t), want (%d, %t, %t)", tt.input, got, exact, err != nil, tt.want.v, tt.want.exact, tt.want.error) + } + }) + } +} diff --git a/go.mod b/go.mod index 82a778d02..e751bcfe0 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,8 @@ require ( github.com/ProtonMail/gopenpgp/v2 v2.9.0 github.com/SheltonZhu/115driver v1.1.1 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible + github.com/antchfx/htmlquery v1.3.5 + github.com/antchfx/xpath v1.3.5 github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.55.7 github.com/blevesearch/bleve/v2 v2.5.2 @@ -104,6 +106,7 @@ require ( github.com/emersion/go-message v0.18.2 // indirect github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect github.com/geoffgarside/ber v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 029e5d514..57964be4c 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,10 @@ github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 h1:8PmGpDEZl9 github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/antchfx/htmlquery v1.3.5 h1:aYthDDClnG2a2xePf6tys/UyyM/kRcsFRm+ifhFKoU0= +github.com/antchfx/htmlquery v1.3.5/go.mod h1:5oyIPIa3ovYGtLqMPNjBF2Uf25NPCKsMjCnQ8lvjaoA= +github.com/antchfx/xpath v1.3.5 h1:PqbXLC3TkfeZyakF5eeh3NTWEbYl4VHNVeufANzDbKQ= +github.com/antchfx/xpath v1.3.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -347,6 +351,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -398,8 +404,6 @@ github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7Fsg github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/halalcloud/golang-sdk-lite v0.0.0-20251006164234-3c629727c499 h1:4ovnBdiGDFi8putQGxhipuuhXItAgh4/YnzufPYkZkQ= -github.com/halalcloud/golang-sdk-lite v0.0.0-20251006164234-3c629727c499/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM= github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38 h1:lsK2GVgI2Ox0NkRpQnN09GBOH7jtsjFK5tcIgxXlLr0= github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=