Skip to content

Commit e283d58

Browse files
lunnylunny
committed
Add migrate repo archiver and packages storage support on command line
1 parent 54d9816 commit e283d58

File tree

9 files changed

+85
-119
lines changed

9 files changed

+85
-119
lines changed

‎cmd/migrate_storage.go

Lines changed: 43 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,11 @@ import (
1212
"code.gitea.io/gitea/models/db"
1313
git_model "code.gitea.io/gitea/models/git"
1414
"code.gitea.io/gitea/models/migrations"
15+
packages_model "code.gitea.io/gitea/models/packages"
1516
repo_model "code.gitea.io/gitea/models/repo"
1617
user_model "code.gitea.io/gitea/models/user"
1718
"code.gitea.io/gitea/modules/log"
19+
packages_module "code.gitea.io/gitea/modules/packages"
1820
"code.gitea.io/gitea/modules/setting"
1921
"code.gitea.io/gitea/modules/storage"
2022

@@ -81,33 +83,58 @@ var CmdMigrateStorage = cli.Command{
8183
}
8284

8385
func migrateAttachments(dstStorage storage.ObjectStorage) error {
84-
return repo_model.IterateAttachment(func(attach *repo_model.Attachment) error {
86+
return db.IterateObjects(db.DefaultContext, func(attach *repo_model.Attachment) error {
8587
_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
8688
return err
8789
})
8890
}
8991

9092
func migrateLFS(dstStorage storage.ObjectStorage) error {
91-
return git_model.IterateLFS(func(mo *git_model.LFSMetaObject) error {
93+
return db.IterateObjects(db.DefaultContext, func(mo *git_model.LFSMetaObject) error {
9294
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
9395
return err
9496
})
9597
}
9698

9799
func migrateAvatars(dstStorage storage.ObjectStorage) error {
98-
return user_model.IterateUser(func(user *user_model.User) error {
100+
return db.IterateObjects(db.DefaultContext, func(user *user_model.User) error {
99101
_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
100102
return err
101103
})
102104
}
103105

104106
func migrateRepoAvatars(dstStorage storage.ObjectStorage) error {
105-
return repo_model.IterateRepository(func(repo *repo_model.Repository) error {
107+
return db.IterateObjects(db.DefaultContext, func(repo *repo_model.Repository) error {
106108
_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
107109
return err
108110
})
109111
}
110112

113+
func migrateRepoArchivers(dstStorage storage.ObjectStorage) error {
114+
return db.IterateObjects(db.DefaultContext, func(archiver *repo_model.RepoArchiver) error {
115+
p, err := archiver.RelativePath()
116+
if err != nil {
117+
return err
118+
}
119+
_, err = storage.Copy(dstStorage, p, storage.RepoArchives, p)
120+
return err
121+
})
122+
}
123+
124+
func migratePackages(dstStorage storage.ObjectStorage) error {
125+
ctx := db.DefaultContext
126+
return db.IterateObjects(ctx, func(pf *packages_model.PackageFile) error {
127+
pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
128+
if err != nil {
129+
return err
130+
}
131+
132+
p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
133+
_, err = storage.Copy(dstStorage, p, storage.RepoAvatars, p)
134+
return err
135+
})
136+
}
137+
111138
func runMigrateStorage(ctx *cli.Context) error {
112139
stdCtx, cancel := installSignals()
113140
defer cancel()
@@ -168,25 +195,21 @@ func runMigrateStorage(ctx *cli.Context) error {
168195
return err
169196
}
170197

198+
migratedMethods := map[string]func(storage.ObjectStorage) error{
199+
"attachments": migrateAttachments,
200+
"lfs": migrateLFS,
201+
"avatars": migrateAvatars,
202+
"repo-avatars": migrateRepoAvatars,
203+
"repo-archivers": migrateRepoArchivers,
204+
"packages": migratePackages,
205+
}
206+
171207
tp := strings.ToLower(ctx.String("type"))
172-
switch tp {
173-
case "attachments":
174-
if err := migrateAttachments(dstStorage); err != nil {
175-
return err
176-
}
177-
case "lfs":
178-
if err := migrateLFS(dstStorage); err != nil {
208+
if m, ok := migratedMethods[tp]; ok {
209+
if err := m(dstStorage); err != nil {
179210
return err
180211
}
181-
case "avatars":
182-
if err := migrateAvatars(dstStorage); err != nil {
183-
return err
184-
}
185-
case "repo-avatars":
186-
if err := migrateRepoAvatars(dstStorage); err != nil {
187-
return err
188-
}
189-
default:
212+
} else {
190213
return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
191214
}
192215

‎models/db/iterate.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package db
6+
7+
import (
8+
"context"
9+
10+
"code.gitea.io/gitea/modules/setting"
11+
)
12+
13+
// IterateObjects iterate all the Bean object
14+
func IterateObjects[Object any](ctx context.Context, f func(repo *Object) error) error {
15+
var start int
16+
batchSize := setting.Database.IterateBufferSize
17+
sess := GetEngine(ctx)
18+
for {
19+
repos := make([]*Object, 0, batchSize)
20+
if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
21+
return err
22+
}
23+
if len(repos) == 0 {
24+
return nil
25+
}
26+
start += len(repos)
27+
28+
for _, repo := range repos {
29+
if err := f(repo); err != nil {
30+
return err
31+
}
32+
}
33+
}
34+
}

‎models/git/lfs.go

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -278,29 +278,6 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6
278278
return committer.Commit()
279279
}
280280

281-
// IterateLFS iterates lfs object
282-
func IterateLFS(f func(mo *LFSMetaObject) error) error {
283-
var start int
284-
const batchSize = 100
285-
e := db.GetEngine(db.DefaultContext)
286-
for {
287-
mos := make([]*LFSMetaObject, 0, batchSize)
288-
if err := e.Limit(batchSize, start).Find(&mos); err != nil {
289-
return err
290-
}
291-
if len(mos) == 0 {
292-
return nil
293-
}
294-
start += len(mos)
295-
296-
for _, mo := range mos {
297-
if err := f(mo); err != nil {
298-
return err
299-
}
300-
}
301-
}
302-
}
303-
304281
// CopyLFS copies LFS data from one repo to another
305282
func CopyLFS(ctx context.Context, newRepo, oldRepo *repo_model.Repository) error {
306283
var lfsObjects []*LFSMetaObject

‎models/repo/attachment.go

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -226,28 +226,6 @@ func DeleteAttachmentsByRelease(releaseID int64) error {
226226
return err
227227
}
228228

229-
// IterateAttachment iterates attachments; it should not be used when Gitea is servicing users.
230-
func IterateAttachment(f func(attach *Attachment) error) error {
231-
var start int
232-
const batchSize = 100
233-
for {
234-
attachments := make([]*Attachment, 0, batchSize)
235-
if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&attachments); err != nil {
236-
return err
237-
}
238-
if len(attachments) == 0 {
239-
return nil
240-
}
241-
start += len(attachments)
242-
243-
for _, attach := range attachments {
244-
if err := f(attach); err != nil {
245-
return err
246-
}
247-
}
248-
}
249-
}
250-
251229
// CountOrphanedAttachments returns the number of bad attachments
252230
func CountOrphanedAttachments() (int64, error) {
253231
return db.GetEngine(db.DefaultContext).Where("(issue_id > 0 and issue_id not in (select id from issue)) or (release_id > 0 and release_id not in (select id from `release`))").

‎models/repo/repo_list.go

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -15,36 +15,12 @@ import (
1515
"code.gitea.io/gitea/models/unit"
1616
user_model "code.gitea.io/gitea/models/user"
1717
"code.gitea.io/gitea/modules/container"
18-
"code.gitea.io/gitea/modules/setting"
1918
"code.gitea.io/gitea/modules/structs"
2019
"code.gitea.io/gitea/modules/util"
2120

2221
"xorm.io/builder"
2322
)
2423

25-
// IterateRepository iterate repositories
26-
func IterateRepository(f func(repo *Repository) error) error {
27-
var start int
28-
batchSize := setting.Database.IterateBufferSize
29-
sess := db.GetEngine(db.DefaultContext)
30-
for {
31-
repos := make([]*Repository, 0, batchSize)
32-
if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
33-
return err
34-
}
35-
if len(repos) == 0 {
36-
return nil
37-
}
38-
start += len(repos)
39-
40-
for _, repo := range repos {
41-
if err := f(repo); err != nil {
42-
return err
43-
}
44-
}
45-
}
46-
}
47-
4824
// FindReposMapByIDs find repos as map
4925
func FindReposMapByIDs(repoIDs []int64, res map[int64]*Repository) error {
5026
return db.GetEngine(db.DefaultContext).In("id", repoIDs).Find(&res)

‎models/user/search.go

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"strings"
1010

1111
"code.gitea.io/gitea/models/db"
12-
"code.gitea.io/gitea/modules/setting"
1312
"code.gitea.io/gitea/modules/structs"
1413
"code.gitea.io/gitea/modules/util"
1514

@@ -125,28 +124,6 @@ func SearchUsers(opts *SearchUserOptions) (users []*User, _ int64, _ error) {
125124
return users, count, sessQuery.Find(&users)
126125
}
127126

128-
// IterateUser iterate users
129-
func IterateUser(f func(user *User) error) error {
130-
var start int
131-
batchSize := setting.Database.IterateBufferSize
132-
for {
133-
users := make([]*User, 0, batchSize)
134-
if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&users); err != nil {
135-
return err
136-
}
137-
if len(users) == 0 {
138-
return nil
139-
}
140-
start += len(users)
141-
142-
for _, user := range users {
143-
if err := f(user); err != nil {
144-
return err
145-
}
146-
}
147-
}
148-
}
149-
150127
// BuildCanSeeUserCondition creates a condition which can be used to restrict results to users/orgs the actor can see
151128
func BuildCanSeeUserCondition(actor *User) builder.Cond {
152129
if actor != nil {

‎modules/packages/content_store.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,21 +27,21 @@ func NewContentStore() *ContentStore {
2727

2828
// Get gets a package blob
2929
func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) {
30-
return s.store.Open(keyToRelativePath(key))
30+
return s.store.Open(KeyToRelativePath(key))
3131
}
3232

3333
// Save stores a package blob
3434
func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error {
35-
_, err := s.store.Save(keyToRelativePath(key), r, size)
35+
_, err := s.store.Save(KeyToRelativePath(key), r, size)
3636
return err
3737
}
3838

3939
// Delete deletes a package blob
4040
func (s *ContentStore) Delete(key BlobHash256Key) error {
41-
return s.store.Delete(keyToRelativePath(key))
41+
return s.store.Delete(KeyToRelativePath(key))
4242
}
4343

44-
// keyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
45-
func keyToRelativePath(key BlobHash256Key) string {
44+
// KeyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
45+
func KeyToRelativePath(key BlobHash256Key) string {
4646
return path.Join(string(key)[0:2], string(key)[2:4], string(key))
4747
}

‎routers/private/mail.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"net/http"
1010
"strconv"
1111

12+
"code.gitea.io/gitea/models/db"
1213
user_model "code.gitea.io/gitea/models/user"
1314
"code.gitea.io/gitea/modules/context"
1415
"code.gitea.io/gitea/modules/json"
@@ -59,7 +60,7 @@ func SendEmail(ctx *context.PrivateContext) {
5960
}
6061
}
6162
} else {
62-
err := user_model.IterateUser(func(user *user_model.User) error {
63+
err := db.IterateObjects(ctx, func(user *user_model.User) error {
6364
if len(user.Email) > 0 && user.IsActive {
6465
emails = append(emails, user.Email)
6566
}

‎services/repository/avatar.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ func DeleteAvatar(repo *repo_model.Repository) error {
9696

9797
// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories
9898
func RemoveRandomAvatars(ctx context.Context) error {
99-
return repo_model.IterateRepository(func(repository *repo_model.Repository) error {
99+
return db.IterateObjects(ctx, func(repository *repo_model.Repository) error {
100100
select {
101101
case <-ctx.Done():
102102
return db.ErrCancelledf("before random avatars removed for %s", repository.FullName())

0 commit comments

Comments
 (0)