fix minio storage iterator path (#24691)

minio storage iterator shows different behavior with local fs iterator.

in local fs storage:

``` go
s.IterateObjects("prefix", func(path,obj)
     println(path) // show "prefix/xxx.file"
})
```

in minio storage:

```go
s.IterateObjects("prefix", func(path,obj)
     println(path) // show "xxx.file"
})
```

I think local fs is correct, minio use wrong `basePath` to trim storage
path prefix.

---------

Co-authored-by: Giteabot <[email protected]>

Author: fuxiaohei

.github/workflows/pull-db-tests.yml

@@ -102,6 +102,13 @@ jobs:
           --health-retries 10
         ports:
           - 6379:6379
+      minio:
+        image: bitnami/minio:2021.3.17
+        env:
+          MINIO_ACCESS_KEY: 123456
+          MINIO_SECRET_KEY: 12345678
+        ports:
+          - "9000:9000"
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4

modules/storage/local.go

@@ -133,8 +133,8 @@ func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
 }
 
 // IterateObjects iterates across the objects in the local storage
-func (l *LocalStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error {
-	dir := l.buildLocalPath(prefix)
+func (l *LocalStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
+	dir := l.buildLocalPath(dirName)
 	return filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
 		if err != nil {
 			return err

modules/storage/local_test.go

@@ -4,8 +4,6 @@
 package storage
 
 import (
-	"bytes"
-	"context"
 	"os"
 	"path/filepath"
 	"testing"
@@ -57,38 +55,5 @@ func TestBuildLocalPath(t *testing.T) {
 
 func TestLocalStorageIterator(t *testing.T) {
 	dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir")
-	l, err := NewLocalStorage(context.Background(), LocalStorageConfig{Path: dir})
-	assert.NoError(t, err)
-
-	testFiles := [][]string{
-		{"a/1.txt", "a1"},
-		{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
-		{"b/1.txt", "b1"},
-		{"b/2.txt", "b2"},
-		{"b/3.txt", "b3"},
-		{"b/x 4.txt", "bx4"},
-	}
-	for _, f := range testFiles {
-		_, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1)
-		assert.NoError(t, err)
-	}
-
-	expectedList := map[string][]string{
-		"a":           {"a/1.txt"},
-		"b":           {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
-		"":            {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
-		"/":           {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
-		"a/b/../../a": {"a/1.txt"},
-	}
-	for dir, expected := range expectedList {
-		count := 0
-		err = l.IterateObjects(dir, func(path string, f Object) error {
-			defer f.Close()
-			assert.Contains(t, expected, path)
-			count++
-			return nil
-		})
-		assert.NoError(t, err)
-		assert.Len(t, expected, count)
-	}
+	testStorageIterator(t, string(LocalStorageType), LocalStorageConfig{Path: dir})
 }

modules/storage/minio.go

@@ -129,7 +129,11 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
 }
 
 func (m *MinioStorage) buildMinioPath(p string) string {
-	return util.PathJoinRelX(m.basePath, p)
+	p = util.PathJoinRelX(m.basePath, p)
+	if p == "." {
+		p = "" // minio doesn't use dot as relative path
+	}
+	return p
 }
 
 // Open opens a file
@@ -224,14 +228,15 @@ func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
 }
 
 // IterateObjects iterates across the objects in the miniostorage
-func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error {
+func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
 	opts := minio.GetObjectOptions{}
 	lobjectCtx, cancel := context.WithCancel(m.ctx)
 	defer cancel()
 
 	basePath := m.basePath
-	if prefix != "" {
-		basePath = m.buildMinioPath(prefix)
+	if dirName != "" {
+		// ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo"
+		basePath = m.buildMinioPath(dirName) + "/"
 	}
 
 	for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{
@@ -244,7 +249,7 @@ func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Ob
 		}
 		if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
 			defer object.Close()
-			return fn(strings.TrimPrefix(mObjInfo.Key, basePath), &minioObject{object})
+			return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object})
 		}(object, fn); err != nil {
 			return convertMinioErr(err)
 		}

modules/storage/minio_test.go

@@ -0,0 +1,18 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package storage
+
+import (
+	"testing"
+)
+
+func TestMinioStorageIterator(t *testing.T) {
+	testStorageIterator(t, string(MinioStorageType), MinioStorageConfig{
+		Endpoint:        "127.0.0.1:9000",
+		AccessKeyID:     "123456",
+		SecretAccessKey: "12345678",
+		Bucket:          "gitea",
+		Location:        "us-east-1",
+	})
+}

modules/storage/storage_test.go

@@ -0,0 +1,49 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package storage
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func testStorageIterator(t *testing.T, typStr string, cfg interface{}) {
+	l, err := NewStorage(typStr, cfg)
+	assert.NoError(t, err)
+
+	testFiles := [][]string{
+		{"a/1.txt", "a1"},
+		{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
+		{"ab/1.txt", "ab1"},
+		{"b/1.txt", "b1"},
+		{"b/2.txt", "b2"},
+		{"b/3.txt", "b3"},
+		{"b/x 4.txt", "bx4"},
+	}
+	for _, f := range testFiles {
+		_, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1)
+		assert.NoError(t, err)
+	}
+
+	expectedList := map[string][]string{
+		"a":           {"a/1.txt"},
+		"b":           {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
+		"":            {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
+		"/":           {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
+		"a/b/../../a": {"a/1.txt"},
+	}
+	for dir, expected := range expectedList {
+		count := 0
+		err = l.IterateObjects(dir, func(path string, f Object) error {
+			defer f.Close()
+			assert.Contains(t, expected, path)
+			count++
+			return nil
+		})
+		assert.NoError(t, err)
+		assert.Len(t, expected, count)
+	}
+}