feat(drivers): add autoindex driver

drivers/all.go

@@ -17,6 +17,7 @@ import (
 	_ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive"
 	_ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_open"
 	_ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_share"
+	_ "github.com/OpenListTeam/OpenList/v4/drivers/autoindex"
 	_ "github.com/OpenListTeam/OpenList/v4/drivers/azure_blob"
 	_ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_netdisk"
 	_ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo"

drivers/autoindex/driver.go

@@ -0,0 +1,169 @@
+package autoindex
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/OpenListTeam/OpenList/v4/drivers/base"
+	"github.com/OpenListTeam/OpenList/v4/internal/driver"
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"github.com/OpenListTeam/OpenList/v4/internal/op"
+	"github.com/antchfx/htmlquery"
+	"github.com/antchfx/xpath"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
+
+type AutoIndex struct {
+	model.Storage
+	Addition
+	itemXPath     *xpath.Expr
+	nameXPath     *xpath.Expr
+	modifiedXPath *xpath.Expr
+	sizeXPath     *xpath.Expr
+	ignores       map[string]any
+}
+
+func (d *AutoIndex) Config() driver.Config {
+	return config
+}
+
+func (d *AutoIndex) GetAddition() driver.Additional {
+	return &d.Addition
+}
+
+func (d *AutoIndex) Init(ctx context.Context) error {
+	var err error
+	d.itemXPath, err = xpath.Compile(d.ItemXPath)
+	if err != nil {
+		return errors.WithMessage(err, "failed to compile Item XPath")
+	}
+	d.nameXPath, err = xpath.Compile(d.NameXPath)
+	if err != nil {
+		return errors.WithMessage(err, "failed to compile Name XPath")
+	}
+	if len(d.ModifiedXPath) > 0 {
+		d.modifiedXPath, err = xpath.Compile(d.ModifiedXPath)
+		if err != nil {
+			return errors.WithMessage(err, "failed to compile Modified XPath")
+		}
+	}
+	if len(d.SizeXPath) > 0 {
+		d.sizeXPath, err = xpath.Compile(d.SizeXPath)
+		if err != nil {
+			return errors.WithMessage(err, "failed to compile Size XPath")
+		}
+	}
+	ignores := strings.Split(d.IgnoreFileNames, "\n")
+	d.ignores = make(map[string]any, len(ignores))
+	for _, i := range ignores {
+		i = strings.TrimSpace(i)
+		if len(i) == 0 {
+			continue
+		}
+		d.ignores[i] = struct{}{}
+	}
+	hasScheme := strings.Contains(d.URL, "://")
+	hasSuffix := strings.HasSuffix(d.URL, "/")
+	if !hasScheme || !hasSuffix {
+		if !hasSuffix {
+			d.URL = d.URL + "/"
+		}
+		if !hasScheme {
+			d.URL = "https://" + d.URL
+		}
+		op.MustSaveDriverStorage(d)
+	}
+	return nil
+}
+
+func (d *AutoIndex) Drop(ctx context.Context) error {
+	return nil
+}
+
+func (d *AutoIndex) GetRoot(ctx context.Context) (model.Obj, error) {
+	return &model.Object{
+		Name:     op.RootName,
+		Path:     d.URL,
+		Modified: d.Modified,
+		Mask:     model.Locked,
+		IsFolder: true,
+	}, nil
+}
+
+func (d *AutoIndex) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
+	res, err := base.RestyClient.R().
+		SetContext(ctx).
+		SetDoNotParseResponse(true).
+		Get(dir.GetPath())
+	if err != nil {
+		return nil, errors.WithMessagef(err, "failed to get url [%s]", dir.GetPath())
+	}
+	defer res.RawResponse.Body.Close()
+	doc, err := htmlquery.Parse(res.RawBody())
+	if err != nil {
+		return nil, errors.WithMessagef(err, "failed to parse [%s]", dir.GetPath())
+	}
+	itemsIter := d.itemXPath.Select(htmlquery.CreateXPathNavigator(doc))
+	var objs []model.Obj
+	for itemsIter.MoveNext() {
+		nameFull, err := parseString(d.nameXPath.Evaluate(itemsIter.Current().Copy()))
+		if err != nil {
+			log.Warnf("skip invalid name evaluating result: %v", err)
+			continue
+		}
+		nameFull = strings.TrimSpace(nameFull)
+		name, isDir := strings.CutSuffix(nameFull, "/")
+		if _, ok := d.ignores[name]; ok {
+			continue
+		}
+		var size int64 = 0
+		exact := false
+		modified := time.Now()
+		if d.sizeXPath != nil {
+			size, exact, err = parseSize(d.sizeXPath.Evaluate(itemsIter.Current().Copy()))
+			if err != nil {
+				log.Errorf("failed to parse size of %s: %v", name, err)
+			}
+		}
+		if d.modifiedXPath != nil {
+			modified, err = parseTime(d.modifiedXPath.Evaluate(itemsIter.Current().Copy()), d.ModifiedTimeFormat)
+			if err != nil {
+				log.Errorf("failed to parse modified time of %s: %v", name, err)
+			}
+		}
+		var o model.Obj = &model.Object{
+			Name:     name,
+			IsFolder: isDir,
+			Path:     dir.GetPath() + nameFull,
+			Modified: modified,
+			Size:     size,
+		}
+		if exact {
+			o = &exactSizeObj{Obj: o}
+		}
+		objs = append(objs, o)
+	}
+	return objs, nil
+}
+
+func (d *AutoIndex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
+	if _, ok := file.(*exactSizeObj); ok || args.Redirect {
+		return &model.Link{URL: file.GetPath()}, nil
+	}
+	res, err := base.RestyClient.R().
+		SetContext(ctx).
+		SetDoNotParseResponse(true).
+		Head(file.GetPath())
+	if err != nil {
+		return nil, errors.WithMessagef(err, "failed to head [%s]", file.GetPath())
+	}
+	_ = res.RawResponse.Body.Close()
+	return &model.Link{
+		URL:           file.GetPath(),
+		ContentLength: res.RawResponse.ContentLength,
+	}, nil
+}
+
+var _ driver.Driver = (*AutoIndex)(nil)

drivers/autoindex/meta.go

@@ -0,0 +1,29 @@
+package autoindex
+
+import (
+	"github.com/OpenListTeam/OpenList/v4/internal/driver"
+	"github.com/OpenListTeam/OpenList/v4/internal/op"
+)
+
+type Addition struct {
+	URL                string `json:"url" required:"true"`
+	ItemXPath          string `json:"item_xpath" required:"true"`
+	NameXPath          string `json:"name_xpath" required:"true"`
+	ModifiedXPath      string `json:"modified_xpath"`
+	SizeXPath          string `json:"size_xpath"`
+	IgnoreFileNames    string `json:"ignore_file_names" type:"text" default:".\n..\nParent Directory\nUp"`
+	ModifiedTimeFormat string `json:"modified_time_format" default:"02-Jan-2006 15:04" help:"Must be based on the time point Mon Jan 2 15:04:05 -0700 MST 2006"`
+}
+
+var config = driver.Config{
+	Name:        "AutoIndex",
+	LocalSort:   true,
+	CheckStatus: true,
+	NoUpload:    true,
+}
+
+func init() {
+	op.RegisterDriver(func() driver.Driver {
+		return &AutoIndex{}
+	})
+}

drivers/autoindex/types.go

@@ -0,0 +1,13 @@
+package autoindex
+
+import (
+	"fmt"
+
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+)
+
+var (
+	errEmptyEvaluateResult = fmt.Errorf("empty result")
+)
+
+type exactSizeObj struct{ model.Obj }

drivers/autoindex/util.go

@@ -0,0 +1,116 @@
+package autoindex
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/antchfx/xpath"
+	"github.com/pkg/errors"
+)
+
+var units = map[string]int64{
+	"":      1,
+	"b":     1,
+	"byte":  1,
+	"bytes": 1,
+	"k":     1 << 10,
+	"kb":    1 << 10,
+	"kib":   1 << 10,
+	"m":     1 << 20,
+	"mb":    1 << 20,
+	"mib":   1 << 20,
+	"g":     1 << 30,
+	"gb":    1 << 30,
+	"gib":   1 << 30,
+	"t":     1 << 40,
+	"tb":    1 << 40,
+	"tib":   1 << 40,
+	"p":     1 << 50,
+	"pb":    1 << 50,
+	"pib":   1 << 50,
+}
+
+func splitUnit(s string) (string, string) {
+	for i := len(s) - 1; i >= 0; i-- {
+		if s[i] >= '0' && s[i] <= '9' {
+			return strings.TrimSpace(s[:i+1]), strings.TrimSpace(s[i+1:])
+		}
+	}
+	return "", s
+}
+
+func parseSize(a any) (int64, bool, error) {
+	// 第二个返回值exact表示大小是否精确
+	if f, ok := a.(float64); ok {
+		return int64(f), false, nil
+	}
+	s, err := parseString(a)
+	if errors.Is(err, errEmptyEvaluateResult) {
+		// 可能是错误，也可能确实大小为0
+		// 如果确实大小为0，大概率不会下载，exact返回false也不会有什么性能损失
+		// 如果是错误，exact返回true会导致本地代理出错，综合来看返回false更好
+		return 0, false, nil
+	}
+	if err != nil {
+		return 0, false, err
+	}
+	s = strings.TrimSpace(s)
+	if s == "-" {
+		return 0, false, nil
+	}
+	nbs, unit := splitUnit(s)
+	mul, ok := units[strings.ToLower(unit)]
+	exact := mul == 1
+	if !ok {
+		mul = 1
+		// 推测无单位，exact应为false
+	}
+	nb, err := strconv.ParseInt(nbs, 10, 64)
+	if err != nil {
+		fnb, err := strconv.ParseFloat(nbs, 64)
+		if err != nil {
+			return 0, false, fmt.Errorf("failed to convert %s to number", nbs)
+		}
+		nb = int64(fnb * float64(mul))
+		exact = false
+	} else {
+		nb = nb * mul
+	}
+	return nb, exact, nil
+}
+
+func parseString(res any) (string, error) {
+	if r, ok := res.(string); ok {
+		if len(r) == 0 {
+			return "", errEmptyEvaluateResult
+		}
+		return r, nil
+	}
+	n, ok := res.(*xpath.NodeIterator)
+	if !ok {
+		return "", fmt.Errorf("unsupported evaluating result")
+	}
+	if !n.MoveNext() {
+		return "", fmt.Errorf("no matched nodes")
+	}
+	ns := n.Current().Value()
+	if len(ns) == 0 {
+		return "", errEmptyEvaluateResult
+	}
+	return ns, nil
+}
+
+func parseTime(res any, format string) (time.Time, error) {
+	s, err := parseString(res)
+	if err != nil {
+		return time.Now(), err
+	}
+	s = strings.TrimSpace(s)
+	t, err := time.Parse(format, s)
+	if err != nil {
+		return time.Now(), errors.WithMessagef(err, "failed to convert %s to time", s)
+	}
+	return t, nil
+}

drivers/autoindex/util_test.go

@@ -0,0 +1,49 @@
+package autoindex
+
+import (
+	"testing"
+)
+
+type wantType struct {
+	v     int64
+	exact bool
+	error bool
+}
+
+func TestParseSize(t *testing.T) {
+	tests := []struct {
+		input string
+		want  wantType
+	}{
+		{"100", wantType{100, true, false}},
+		{"1k", wantType{1024, false, false}},
+		{"1kb", wantType{1024, false, false}},
+		{"1K", wantType{1024, false, false}},      // case insensitive
+		{"1.5m", wantType{1572864, false, false}}, // 1.5 * 1024^2
+		{"500 bytes", wantType{500, true, false}},
+		{"-", wantType{0, false, false}},
+		{"", wantType{0, false, false}},
+		{"abc", wantType{0, false, true}},
+		{"1.5GB", wantType{1610612736, false, false}},    // 1.5 * 1024^3
+		{"2t", wantType{2199023255552, false, false}},    // 2 * 1024^4
+		{"1p", wantType{1125899906842624, false, false}}, // 1 * 1024^5
+		{"0", wantType{0, true, false}},
+		{"  100  ", wantType{100, true, false}}, // trimmed
+		{"100b", wantType{100, true, false}},
+		{"1gib", wantType{1073741824, false, false}}, // 1024^3
+		{"1z", wantType{1, false, false}},            // invalid unit, mul=1
+		{"1.5", wantType{1, false, false}},           // float without unit, truncated
+		{"2.7k", wantType{2764, false, false}},       // 2.7 * 1024 truncated
+		{"1.0g", wantType{1073741824, false, false}}, // 1.0 * 1024^3
+		{"invalid", wantType{0, false, true}},
+		{"123xyz", wantType{123, false, false}}, // unit not found, mul=1
+	}
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			got, exact, err := parseSize(tt.input)
+			if got != tt.want.v || exact != tt.want.exact || (err != nil) != tt.want.error {
+				t.Errorf("ParseSize(%q) = (%d, %t, %t), want (%d, %t, %t)", tt.input, got, exact, err != nil, tt.want.v, tt.want.exact, tt.want.error)
+			}
+		})
+	}
+}