Skip to content

Commit 40011bb

Browse files
lunnylunny
committed
Merge main branch
2 parents 381624b + 1f14609 commit 40011bb

File tree

202 files changed

+1179
-1160
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

202 files changed

+1179
-1160
lines changed

‎cmd/main_test.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package cmd
6+
7+
import (
8+
"testing"
9+
10+
"code.gitea.io/gitea/models/unittest"
11+
"code.gitea.io/gitea/modules/setting"
12+
)
13+
14+
func init() {
15+
setting.SetCustomPathAndConf("", "", "")
16+
setting.LoadForTest()
17+
}
18+
19+
func TestMain(m *testing.M) {
20+
unittest.MainTest(m, &unittest.TestOptions{
21+
GiteaRootPath: "..",
22+
})
23+
}

‎cmd/migrate_storage.go

Lines changed: 48 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,11 @@ import (
1212
"code.gitea.io/gitea/models/db"
1313
git_model "code.gitea.io/gitea/models/git"
1414
"code.gitea.io/gitea/models/migrations"
15+
packages_model "code.gitea.io/gitea/models/packages"
1516
repo_model "code.gitea.io/gitea/models/repo"
1617
user_model "code.gitea.io/gitea/models/user"
1718
"code.gitea.io/gitea/modules/log"
19+
packages_module "code.gitea.io/gitea/modules/packages"
1820
"code.gitea.io/gitea/modules/setting"
1921
"code.gitea.io/gitea/modules/storage"
2022

@@ -25,13 +27,13 @@ import (
2527
var CmdMigrateStorage = cli.Command{
2628
Name: "migrate-storage",
2729
Usage: "Migrate the storage",
28-
Description: "This is a command for migrating storage.",
30+
Description: "Copies stored files from storage configured in app.ini to parameter-configured storage",
2931
Action: runMigrateStorage,
3032
Flags: []cli.Flag{
3133
cli.StringFlag{
3234
Name: "type, t",
3335
Value: "",
34-
Usage: "Kinds of files to migrate, currently only 'attachments' is supported",
36+
Usage: "Type of stored files to copy. Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages'",
3537
},
3638
cli.StringFlag{
3739
Name: "storage, s",
@@ -80,34 +82,53 @@ var CmdMigrateStorage = cli.Command{
8082
},
8183
}
8284

83-
func migrateAttachments(dstStorage storage.ObjectStorage) error {
84-
return repo_model.IterateAttachment(func(attach *repo_model.Attachment) error {
85+
func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
86+
return db.IterateObjects(ctx, func(attach *repo_model.Attachment) error {
8587
_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
8688
return err
8789
})
8890
}
8991

90-
func migrateLFS(dstStorage storage.ObjectStorage) error {
91-
return git_model.IterateLFS(func(mo *git_model.LFSMetaObject) error {
92+
func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
93+
return db.IterateObjects(ctx, func(mo *git_model.LFSMetaObject) error {
9294
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
9395
return err
9496
})
9597
}
9698

97-
func migrateAvatars(dstStorage storage.ObjectStorage) error {
98-
return user_model.IterateUser(func(user *user_model.User) error {
99+
func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
100+
return db.IterateObjects(ctx, func(user *user_model.User) error {
99101
_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
100102
return err
101103
})
102104
}
103105

104-
func migrateRepoAvatars(dstStorage storage.ObjectStorage) error {
105-
return repo_model.IterateRepository(func(repo *repo_model.Repository) error {
106+
func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
107+
return db.IterateObjects(ctx, func(repo *repo_model.Repository) error {
106108
_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
107109
return err
108110
})
109111
}
110112

113+
func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
114+
return db.IterateObjects(ctx, func(archiver *repo_model.RepoArchiver) error {
115+
p, err := archiver.RelativePath()
116+
if err != nil {
117+
return err
118+
}
119+
_, err = storage.Copy(dstStorage, p, storage.RepoArchives, p)
120+
return err
121+
})
122+
}
123+
124+
func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
125+
return db.IterateObjects(ctx, func(pb *packages_model.PackageBlob) error {
126+
p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
127+
_, err := storage.Copy(dstStorage, p, storage.Packages, p)
128+
return err
129+
})
130+
}
131+
111132
func runMigrateStorage(ctx *cli.Context) error {
112133
stdCtx, cancel := installSignals()
113134
defer cancel()
@@ -127,8 +148,6 @@ func runMigrateStorage(ctx *cli.Context) error {
127148
return err
128149
}
129150

130-
goCtx := context.Background()
131-
132151
if err := storage.Init(); err != nil {
133152
return err
134153
}
@@ -145,13 +164,13 @@ func runMigrateStorage(ctx *cli.Context) error {
145164
return nil
146165
}
147166
dstStorage, err = storage.NewLocalStorage(
148-
goCtx,
167+
stdCtx,
149168
storage.LocalStorageConfig{
150169
Path: p,
151170
})
152171
case string(storage.MinioStorageType):
153172
dstStorage, err = storage.NewMinioStorage(
154-
goCtx,
173+
stdCtx,
155174
storage.MinioStorageConfig{
156175
Endpoint: ctx.String("minio-endpoint"),
157176
AccessKeyID: ctx.String("minio-access-key-id"),
@@ -162,35 +181,29 @@ func runMigrateStorage(ctx *cli.Context) error {
162181
UseSSL: ctx.Bool("minio-use-ssl"),
163182
})
164183
default:
165-
return fmt.Errorf("Unsupported storage type: %s", ctx.String("storage"))
184+
return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
166185
}
167186
if err != nil {
168187
return err
169188
}
170189

190+
migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{
191+
"attachments": migrateAttachments,
192+
"lfs": migrateLFS,
193+
"avatars": migrateAvatars,
194+
"repo-avatars": migrateRepoAvatars,
195+
"repo-archivers": migrateRepoArchivers,
196+
"packages": migratePackages,
197+
}
198+
171199
tp := strings.ToLower(ctx.String("type"))
172-
switch tp {
173-
case "attachments":
174-
if err := migrateAttachments(dstStorage); err != nil {
175-
return err
176-
}
177-
case "lfs":
178-
if err := migrateLFS(dstStorage); err != nil {
179-
return err
180-
}
181-
case "avatars":
182-
if err := migrateAvatars(dstStorage); err != nil {
183-
return err
184-
}
185-
case "repo-avatars":
186-
if err := migrateRepoAvatars(dstStorage); err != nil {
200+
if m, ok := migratedMethods[tp]; ok {
201+
if err := m(stdCtx, dstStorage); err != nil {
187202
return err
188203
}
189-
default:
190-
return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
204+
log.Info("%s files have successfully been copied to the new storage.", tp)
205+
return nil
191206
}
192207

193-
log.Warn("All files have been copied to the new placement but old files are still on the original placement.")
194-
195-
return nil
208+
return fmt.Errorf("unsupported storage: %s", ctx.String("type"))
196209
}

‎cmd/migrate_storage_test.go

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package cmd
6+
7+
import (
8+
"context"
9+
"os"
10+
"strings"
11+
"testing"
12+
13+
"code.gitea.io/gitea/models/packages"
14+
"code.gitea.io/gitea/models/unittest"
15+
user_model "code.gitea.io/gitea/models/user"
16+
packages_module "code.gitea.io/gitea/modules/packages"
17+
"code.gitea.io/gitea/modules/storage"
18+
packages_service "code.gitea.io/gitea/services/packages"
19+
20+
"github.com/stretchr/testify/assert"
21+
)
22+
23+
func TestMigratePackages(t *testing.T) {
24+
assert.NoError(t, unittest.PrepareTestDatabase())
25+
26+
creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
27+
28+
content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n"
29+
buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024)
30+
assert.NoError(t, err)
31+
defer buf.Close()
32+
33+
v, f, err := packages_service.CreatePackageAndAddFile(&packages_service.PackageCreationInfo{
34+
PackageInfo: packages_service.PackageInfo{
35+
Owner: creator,
36+
PackageType: packages.TypeGeneric,
37+
Name: "test",
38+
Version: "1.0.0",
39+
},
40+
Creator: creator,
41+
SemverCompatible: true,
42+
VersionProperties: map[string]string{},
43+
}, &packages_service.PackageFileCreationInfo{
44+
PackageFileInfo: packages_service.PackageFileInfo{
45+
Filename: "a.go",
46+
},
47+
Data: buf,
48+
IsLead: true,
49+
})
50+
assert.NoError(t, err)
51+
assert.NotNil(t, v)
52+
assert.NotNil(t, f)
53+
54+
ctx := context.Background()
55+
56+
p, err := os.MkdirTemp(os.TempDir(), "migrated_packages")
57+
assert.NoError(t, err)
58+
59+
dstStorage, err := storage.NewLocalStorage(
60+
ctx,
61+
storage.LocalStorageConfig{
62+
Path: p,
63+
})
64+
assert.NoError(t, err)
65+
66+
err = migratePackages(ctx, dstStorage)
67+
assert.NoError(t, err)
68+
69+
entries, err := os.ReadDir(p)
70+
assert.NoError(t, err)
71+
assert.EqualValues(t, 2, len(entries))
72+
assert.EqualValues(t, "01", entries[0].Name())
73+
assert.EqualValues(t, "tmp", entries[1].Name())
74+
}

‎integrations/admin_user_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ func makeRequest(t *testing.T, formData user_model.User, headerCode int) {
6161
})
6262

6363
session.MakeRequest(t, req, headerCode)
64-
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: formData.ID}).(*user_model.User)
64+
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: formData.ID})
6565
assert.Equal(t, formData.Name, user.Name)
6666
assert.Equal(t, formData.LoginName, user.LoginName)
6767
assert.Equal(t, formData.Email, user.Email)

‎integrations/api_admin_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ func TestAPIAdminCreateAndDeleteSSHKey(t *testing.T) {
2222
defer prepareTestEnv(t)()
2323
// user1 is an admin user
2424
session := loginUser(t, "user1")
25-
keyOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user2"}).(*user_model.User)
25+
keyOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user2"})
2626

2727
token := getTokenForLoggedInUser(t, session)
2828
urlStr := fmt.Sprintf("/api/v1/admin/users/%s/keys?token=%s", keyOwner.Name, token)
@@ -194,7 +194,7 @@ func TestAPIEditUser(t *testing.T) {
194194
json.Unmarshal(resp.Body.Bytes(), &errMap)
195195
assert.EqualValues(t, "email is not allowed to be empty string", errMap["message"].(string))
196196

197-
user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{LoginName: "user2"}).(*user_model.User)
197+
user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{LoginName: "user2"})
198198
assert.False(t, user2.IsRestricted)
199199
bTrue := true
200200
req = NewRequestWithJSON(t, "PATCH", urlStr, api.EditUserOption{
@@ -205,6 +205,6 @@ func TestAPIEditUser(t *testing.T) {
205205
Restricted: &bTrue,
206206
})
207207
session.MakeRequest(t, req, http.StatusOK)
208-
user2 = unittest.AssertExistsAndLoadBean(t, &user_model.User{LoginName: "user2"}).(*user_model.User)
208+
user2 = unittest.AssertExistsAndLoadBean(t, &user_model.User{LoginName: "user2"})
209209
assert.True(t, user2.IsRestricted)
210210
}

‎integrations/api_comment_test.go

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,10 @@ func TestAPIListRepoComments(t *testing.T) {
2424
defer prepareTestEnv(t)()
2525

2626
comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{},
27-
unittest.Cond("type = ?", issues_model.CommentTypeComment)).(*issues_model.Comment)
28-
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: comment.IssueID}).(*issues_model.Issue)
29-
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID}).(*repo_model.Repository)
30-
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User)
27+
unittest.Cond("type = ?", issues_model.CommentTypeComment))
28+
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: comment.IssueID})
29+
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID})
30+
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID})
3131

3232
session := loginUser(t, repoOwner.Name)
3333
link, _ := url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/issues/comments", repoOwner.Name, repo.Name))
@@ -70,10 +70,10 @@ func TestAPIListIssueComments(t *testing.T) {
7070
defer prepareTestEnv(t)()
7171

7272
comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{},
73-
unittest.Cond("type = ?", issues_model.CommentTypeComment)).(*issues_model.Comment)
74-
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: comment.IssueID}).(*issues_model.Issue)
75-
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID}).(*repo_model.Repository)
76-
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User)
73+
unittest.Cond("type = ?", issues_model.CommentTypeComment))
74+
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: comment.IssueID})
75+
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID})
76+
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID})
7777

7878
session := loginUser(t, repoOwner.Name)
7979
req := NewRequestf(t, "GET", "/api/v1/repos/%s/%s/issues/%d/comments",
@@ -91,9 +91,9 @@ func TestAPICreateComment(t *testing.T) {
9191
defer prepareTestEnv(t)()
9292
const commentBody = "Comment body"
9393

94-
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{}).(*issues_model.Issue)
95-
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID}).(*repo_model.Repository)
96-
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User)
94+
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{})
95+
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID})
96+
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID})
9797

9898
session := loginUser(t, repoOwner.Name)
9999
token := getTokenForLoggedInUser(t, session)
@@ -113,10 +113,10 @@ func TestAPICreateComment(t *testing.T) {
113113
func TestAPIGetComment(t *testing.T) {
114114
defer prepareTestEnv(t)()
115115

116-
comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 2}).(*issues_model.Comment)
116+
comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 2})
117117
assert.NoError(t, comment.LoadIssue())
118-
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: comment.Issue.RepoID}).(*repo_model.Repository)
119-
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User)
118+
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: comment.Issue.RepoID})
119+
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID})
120120

121121
session := loginUser(t, repoOwner.Name)
122122
token := getTokenForLoggedInUser(t, session)
@@ -142,10 +142,10 @@ func TestAPIEditComment(t *testing.T) {
142142
const newCommentBody = "This is the new comment body"
143143

144144
comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{},
145-
unittest.Cond("type = ?", issues_model.CommentTypeComment)).(*issues_model.Comment)
146-
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: comment.IssueID}).(*issues_model.Issue)
147-
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID}).(*repo_model.Repository)
148-
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User)
145+
unittest.Cond("type = ?", issues_model.CommentTypeComment))
146+
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: comment.IssueID})
147+
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID})
148+
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID})
149149

150150
session := loginUser(t, repoOwner.Name)
151151
token := getTokenForLoggedInUser(t, session)
@@ -167,10 +167,10 @@ func TestAPIDeleteComment(t *testing.T) {
167167
defer prepareTestEnv(t)()
168168

169169
comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{},
170-
unittest.Cond("type = ?", issues_model.CommentTypeComment)).(*issues_model.Comment)
171-
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: comment.IssueID}).(*issues_model.Issue)
172-
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID}).(*repo_model.Repository)
173-
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User)
170+
unittest.Cond("type = ?", issues_model.CommentTypeComment))
171+
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: comment.IssueID})
172+
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID})
173+
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID})
174174

175175
session := loginUser(t, repoOwner.Name)
176176
token := getTokenForLoggedInUser(t, session)
@@ -185,9 +185,9 @@ func TestAPIListIssueTimeline(t *testing.T) {
185185
defer prepareTestEnv(t)()
186186

187187
// load comment
188-
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 1}).(*issues_model.Issue)
189-
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID}).(*repo_model.Repository)
190-
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User)
188+
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 1})
189+
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID})
190+
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID})
191191

192192
// make request
193193
session := loginUser(t, repoOwner.Name)

0 commit comments

Comments
 (0)