Process updating user in batches

Signed-off-by: Masudur Rahman <[email protected]>

Author: masudur-rahman

models/migrations/v115.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 
 	"xorm.io/xorm"
@@ -19,71 +20,75 @@ import (
 func renameExistingUserAvatarName(x *xorm.Engine) error {
 	sess := x.NewSession()
 	defer sess.Close()
-	if err := sess.Begin(); err != nil {
-		return err
-	}
 
 	type User struct {
 		ID     int64 `xorm:"pk autoincr"`
 		Avatar string
 	}
-
-	users := make([]User, 0)
-	if err := sess.Find(&users); err != nil {
-		return err
-	}
-
 	deleteList := make(map[string]struct{})
-	sessionOpCount := 0
-	for _, user := range users {
-		oldAvatar := user.Avatar
-		newAvatar := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%d-%s", user.ID, user.Avatar))))
-		if _, err := os.Stat(filepath.Join(setting.AvatarUploadPath, oldAvatar)); err != nil {
-			continue
+	start := 0
+	for {
+		if err := sess.Begin(); err != nil {
+			return fmt.Errorf("session.Begin: %v", err)
 		}
-
-		fr, err := os.Open(filepath.Join(setting.AvatarUploadPath, oldAvatar))
-		if err != nil {
-			_ = commitSession(sess)
-			return fmt.Errorf("os.Open: %v", err)
+		users := make([]User, 0, 50)
+		if err := sess.Table("user").Asc("id").Limit(50, start).Find(&users); err != nil {
+			return fmt.Errorf("select users from id [%d]: %v", start, err)
 		}
-		defer fr.Close()
-
-		fw, err := os.Create(filepath.Join(setting.AvatarUploadPath, newAvatar))
-		if err != nil {
-			_ = commitSession(sess)
-			return fmt.Errorf("os.Create: %v", err)
+		if len(users) == 0 {
+			break
 		}
-		defer fw.Close()
+		log.Info("select users [%d - %d]", start, start+len(users))
+		start += 50
 
-		if _, err := io.Copy(fw, fr); err != nil {
-			_ = commitSession(sess)
-			return fmt.Errorf("io.Copy: %v", err)
-		}
+		for _, user := range users {
+			oldAvatar := user.Avatar
+			newAvatar := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%d-%s", user.ID, user.Avatar))))
+			if _, err := os.Stat(filepath.Join(setting.AvatarUploadPath, oldAvatar)); err != nil {
+				log.Warn("os.Stat: %v", err)
+				continue
+			}
 
-		user.Avatar = newAvatar
-		if _, err := sess.ID(user.ID).Update(&user); err != nil {
-			return fmt.Errorf("user table update: %v", err)
-		}
+			fr, err := os.Open(filepath.Join(setting.AvatarUploadPath, oldAvatar))
+			if err != nil {
+				if err := commitSession(sess); err != nil {
+					return fmt.Errorf("commit session: %v", err)
+				}
+				return fmt.Errorf("os.Open: %v", err)
+			}
+			defer fr.Close()
 
-		deleteList[filepath.Join(setting.AvatarUploadPath, oldAvatar)] = struct{}{}
+			fw, err := os.Create(filepath.Join(setting.AvatarUploadPath, newAvatar))
+			if err != nil {
+				if err := commitSession(sess); err != nil {
+					return fmt.Errorf("commit session: %v", err)
+				}
+				return fmt.Errorf("os.Create: %v", err)
+			}
+			defer fw.Close()
 
-		if sessionOpCount++; sessionOpCount >= 100 {
-			if err := commitSession(sess); err != nil {
-				return err
+			if _, err := io.Copy(fw, fr); err != nil {
+				if err := commitSession(sess); err != nil {
+					return fmt.Errorf("commit session: %v", err)
+				}
+				return fmt.Errorf("io.Copy: %v", err)
 			}
-			sessionOpCount = 0
-			if err := sess.Begin(); err != nil {
-				return err
+
+			user.Avatar = newAvatar
+			if _, err := sess.ID(user.ID).Update(&user); err != nil {
+				return fmt.Errorf("user table update: %v", err)
 			}
+
+			deleteList[filepath.Join(setting.AvatarUploadPath, oldAvatar)] = struct{}{}
+		}
+		if err := commitSession(sess); err != nil {
+			return fmt.Errorf("commit session: %v", err)
 		}
 	}
-	if err := commitSession(sess); err != nil {
-		return err
-	}
+
 	for file := range deleteList {
 		if err := os.Remove(file); err != nil {
-			return fmt.Errorf("os.Remove: %v", err)
+			log.Warn("os.Remove: %v", err)
 		}
 	}
 	return nil
@@ -92,7 +97,7 @@ func renameExistingUserAvatarName(x *xorm.Engine) error {
 func commitSession(sess *xorm.Session) error {
 	if err := sess.Commit(); err != nil {
 		_ = sess.Rollback()
-		return fmt.Errorf("db update: %v", err)
+		return err
 	}
 	return nil
 }