Merge branch 'master' into slight-performance-improvement-for-last-commit-cache

Author: zeripath

Makefile

@@ -210,9 +210,10 @@ git-check:
 .PHONY: node-check
 node-check:
 	$(eval NODE_VERSION := $(shell printf "%03d%03d%03d" $(shell node -v | cut -c2- | tr '.' ' ');))
+	$(eval MIN_NODE_VER_FMT := $(shell printf "%g.%g.%g" $(shell echo $(MIN_NODE_VERSION) | grep -o ...)))
 	$(eval NPM_MISSING := $(shell hash npm > /dev/null 2>&1 || echo 1))
 	@if [ "$(NODE_VERSION)" -lt "$(MIN_NODE_VERSION)" -o "$(NPM_MISSING)" = "1" ]; then \
-		echo "Gitea requires Node.js 10 or greater and npm to build. You can get it at https://nodejs.org/en/download/"; \
+		echo "Gitea requires Node.js $(MIN_NODE_VER_FMT) or greater and npm to build. You can get it at https://nodejs.org/en/download/"; \
 		exit 1; \
 	fi
 

contrib/pr/checkout.go

@@ -114,7 +114,7 @@ func runPR() {
 
 	log.Printf("[PR] Setting up router\n")
 	//routers.GlobalInit()
-	external.RegisterParsers()
+	external.RegisterRenderers()
 	markup.Init()
 	c := routes.NormalRoutes()
 

docs/content/doc/advanced/customizing-gitea.en-us.md

@@ -42,7 +42,7 @@ environment variable; this can be used to override the default path to something
 `GITEA_CUSTOM` might, for example, be set by an init script. You can check whether the value
 is set under the "Configuration" tab on the site administration page.
 
-- [List of Environment Variables](https://docs.gitea.io/en-us/specific-variables/)
+- [List of Environment Variables](https://docs.gitea.io/en-us/environment-variables/)
 
 **Note:** Gitea must perform a full restart to see configuration changes.
 

docs/content/doc/developers/api-usage.en-us.md

@@ -79,6 +79,9 @@ API Reference guide is auto-generated by swagger and available on:
 or on
 [gitea demo instance](https://try.gitea.io/api/swagger)
 
+The OpenAPI document is at:
+`https://gitea.your.host/swagger.v1.json`
+
 ## Listing your issued tokens via the API
 
 As mentioned in

docs/content/doc/help/seek-help.en-us.md

@@ -33,4 +33,4 @@ If you found a bug, please create an [issue on GitHub](https://github.com/go-git
 
 ## Chinese Support
 
-Support for the Chinese language is provided at [gocn.vip](https://gocn.vip/topic/gitea).
+Support for the Chinese language is provided at [Our discourse](https://discourse.gitea.io/c/5-category/5).

docs/content/doc/help/seek-help.zh-cn.md

@@ -18,6 +18,6 @@ menu:
 如果您在使用或者开发过程中遇到问题，请到以下渠道咨询：
 
 - 到[Github issue](https://github.com/go-gitea/gitea/issues)提问(因为项目维护人员来自世界各地，为保证沟通顺畅，请使用英文提问)
-- 中文问题到[gocn.vip](https://gocn.vip/topic/gitea)提问
+- 中文问题到 [Gitea 论坛](https://discourse.gitea.io/c/5-category/5)提问
 - 访问 [Discord server - 英文](https://discord.gg/Gitea)
 - 加入 QQ群 328432459 获得进一步的支持

integrations/git_clone_wiki_test.go

@@ -0,0 +1,51 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"path/filepath"
+	"testing"
+
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func assertFileExist(t *testing.T, p string) {
+	exist, err := util.IsExist(p)
+	assert.NoError(t, err)
+	assert.True(t, exist)
+}
+
+func assertFileEqual(t *testing.T, p string, content []byte) {
+	bs, err := ioutil.ReadFile(p)
+	assert.NoError(t, err)
+	assert.EqualValues(t, content, bs)
+}
+
+func TestRepoCloneWiki(t *testing.T) {
+	onGiteaRun(t, func(t *testing.T, u *url.URL) {
+		defer prepareTestEnv(t)()
+
+		dstPath, err := ioutil.TempDir("", "clone_wiki")
+		assert.NoError(t, err)
+
+		r := fmt.Sprintf("%suser2/repo1.wiki.git", u.String())
+		u, _ = url.Parse(r)
+		u.User = url.UserPassword("user2", userPassword)
+		t.Run("Clone", func(t *testing.T) {
+			assert.NoError(t, git.CloneWithArgs(context.Background(), u.String(), dstPath, allowLFSFilters(), git.CloneRepoOptions{}))
+			assertFileEqual(t, filepath.Join(dstPath, "Home.md"), []byte("# Home page\n\nThis is the home page!\n"))
+			assertFileExist(t, filepath.Join(dstPath, "Page-With-Image.md"))
+			assertFileExist(t, filepath.Join(dstPath, "Page-With-Spaced-Name.md"))
+			assertFileExist(t, filepath.Join(dstPath, "images"))
+			assertFileExist(t, filepath.Join(dstPath, "jpeg.jpg"))
+		})
+	})
+}

models/avatar.go

@@ -81,7 +81,7 @@ func LibravatarURL(email string) (*url.URL, error) {
 }
 
 // HashedAvatarLink returns an avatar link for a provided email
-func HashedAvatarLink(email string) string {
+func HashedAvatarLink(email string, size int) string {
 	lowerEmail := strings.ToLower(strings.TrimSpace(email))
 	sum := fmt.Sprintf("%x", md5.Sum([]byte(lowerEmail)))
 	_, _ = cache.GetString("Avatar:"+sum, func() (string, error) {
@@ -108,6 +108,9 @@ func HashedAvatarLink(email string) string {
 		}
 		return lowerEmail, nil
 	})
+	if size > 0 {
+		return setting.AppSubURL + "/avatar/" + url.PathEscape(sum) + "?size=" + strconv.Itoa(size)
+	}
 	return setting.AppSubURL + "/avatar/" + url.PathEscape(sum)
 }
 
@@ -129,7 +132,7 @@ func SizedAvatarLink(email string, size int) string {
 		// This is the slow path that would need to call LibravatarURL() which
 		// does DNS lookups. Avoid it by issuing a redirect so we don't block
 		// the template render with network requests.
-		return HashedAvatarLink(email)
+		return HashedAvatarLink(email, size)
 	} else if !setting.DisableGravatar {
 		// copy GravatarSourceURL, because we will modify its Path.
 		copyOfGravatarSourceURL := *setting.GravatarSourceURL

models/issue_comment.go

@@ -16,6 +16,7 @@ import (
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/references"
 	"code.gitea.io/gitea/modules/structs"
@@ -1178,8 +1179,13 @@ func findCodeComments(e Engine, opts FindCommentsOptions, issue *Issue, currentU
 			return nil, err
 		}
 
-		comment.RenderedContent = string(markdown.Render([]byte(comment.Content), issue.Repo.Link(),
-			issue.Repo.ComposeMetas()))
+		var err error
+		if comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+			URLPrefix: issue.Repo.Link(),
+			Metas:     issue.Repo.ComposeMetas(),
+		}, comment.Content); err != nil {
+			return nil, err
+		}
 	}
 	return comments[:n], nil
 }

models/migrations/v156.go

@@ -88,6 +88,7 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error {
 				repo = new(Repository)
 				has, err := sess.ID(release.RepoID).Get(repo)
 				if err != nil {
+					log.Error("Error whilst loading repository[%d] for release[%d] with tag name %s", release.RepoID, release.ID, release.TagName)
 					return err
 				} else if !has {
 					log.Warn("Release[%d] is orphaned and refers to non-existing repository %d", release.ID, release.RepoID)
@@ -99,28 +100,37 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error {
 					// v120.go migration may not have been run correctly - we'll just replicate it here
 					// because this appears to be a common-ish problem.
 					if _, err := sess.Exec("UPDATE repository SET owner_name = (SELECT name FROM `user` WHERE `user`.id = repository.owner_id)"); err != nil {
+						log.Error("Error whilst updating repository[%d] owner name", repo.ID)
 						return err
 					}
 
 					if _, err := sess.ID(release.RepoID).Get(repo); err != nil {
+						log.Error("Error whilst loading repository[%d] for release[%d] with tag name %s", release.RepoID, release.ID, release.TagName)
 						return err
 					}
 				}
 				gitRepo, err = git.OpenRepository(repoPath(repo.OwnerName, repo.Name))
 				if err != nil {
+					log.Error("Error whilst opening git repo for %-v", repo)
 					return err
 				}
 			}
 
 			commit, err := gitRepo.GetTagCommit(release.TagName)
 			if err != nil {
+				if git.IsErrNotExist(err) {
+					log.Warn("Unable to find commit %s for Tag: %s in %-v. Cannot update publisher ID.", err.(*git.ErrNotExist).ID, release.TagName, repo)
+					continue
+				}
+				log.Error("Error whilst getting commit for Tag: %s in %-v.", release.TagName, repo)
 				return fmt.Errorf("GetTagCommit: %v", err)
 			}
 
 			if user == nil || !strings.EqualFold(user.Email, commit.Author.Email) {
 				user = new(User)
 				_, err = sess.Where("email=?", commit.Author.Email).Get(user)
 				if err != nil {
+					log.Error("Error whilst getting commit author by email: %s for Tag: %s in %-v.", commit.Author.Email, release.TagName, repo)
 					return err
 				}
 
@@ -133,6 +143,7 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error {
 
 			release.PublisherID = user.ID
 			if _, err := sess.ID(release.ID).Cols("publisher_id").Update(release); err != nil {
+				log.Error("Error whilst updating publisher[%d] for release[%d] with tag name %s", release.PublisherID, release.ID, release.TagName)
 				return err
 			}
 		}

models/repo.go

@@ -863,7 +863,10 @@ func (repo *Repository) getUsersWithAccessMode(e Engine, mode AccessMode) (_ []*
 
 // DescriptionHTML does special handles to description and return HTML string.
 func (repo *Repository) DescriptionHTML() template.HTML {
-	desc, err := markup.RenderDescriptionHTML([]byte(repo.Description), repo.HTMLURL(), repo.ComposeMetas())
+	desc, err := markup.RenderDescriptionHTML(&markup.RenderContext{
+		URLPrefix: repo.HTMLURL(),
+		Metas:     repo.ComposeMetas(),
+	}, repo.Description)
 	if err != nil {
 		log.Error("Failed to render description for %s (ID: %d): %v", repo.Name, repo.ID, err)
 		return template.HTML(markup.Sanitize(repo.Description))

models/repo_generate.go

@@ -5,13 +5,14 @@
 package models
 
 import (
+	"bufio"
+	"bytes"
 	"strconv"
 	"strings"
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/storage"
-	"code.gitea.io/gitea/modules/util"
 
 	"github.com/gobwas/glob"
 )
@@ -49,9 +50,9 @@ func (gt GiteaTemplate) Globs() []glob.Glob {
 	}
 
 	gt.globs = make([]glob.Glob, 0)
-	lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n")
-	for _, line := range lines {
-		line = strings.TrimSpace(line)
+	scanner := bufio.NewScanner(bytes.NewReader(gt.Content))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
 		if line == "" || strings.HasPrefix(line, "#") {
 			continue
 		}

models/user_avatar.go

@@ -82,14 +82,19 @@ func (u *User) RealSizedAvatarLink(size int) string {
 		if u.Avatar == "" {
 			return DefaultAvatarLink()
 		}
+		if size > 0 {
+			return setting.AppSubURL + "/avatars/" + u.Avatar + "?size=" + strconv.Itoa(size)
+		}
 		return setting.AppSubURL + "/avatars/" + u.Avatar
 	case setting.DisableGravatar, setting.OfflineMode:
 		if u.Avatar == "" {
 			if err := u.GenerateRandomAvatar(); err != nil {
 				log.Error("GenerateRandomAvatar: %v", err)
 			}
 		}
-
+		if size > 0 {
+			return setting.AppSubURL + "/avatars/" + u.Avatar + "?size=" + strconv.Itoa(size)
+		}
 		return setting.AppSubURL + "/avatars/" + u.Avatar
 	}
 	return SizedAvatarLink(u.AvatarEmail, size)

modules/charset/charset.go

@@ -7,6 +7,8 @@ package charset
 import (
 	"bytes"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"strings"
 	"unicode/utf8"
 
@@ -21,6 +23,33 @@ import (
 // UTF8BOM is the utf-8 byte-order marker
 var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
 
+// ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible
+func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
+	var buf = make([]byte, 2048)
+	n, err := rd.Read(buf)
+	if err != nil {
+		return rd
+	}
+
+	charsetLabel, err := DetectEncoding(buf[:n])
+	if err != nil || charsetLabel == "UTF-8" {
+		return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return io.MultiReader(bytes.NewReader(buf[:n]), rd)
+	}
+
+	return transform.NewReader(
+		io.MultiReader(
+			bytes.NewReader(RemoveBOMIfPresent(buf[:n])),
+			rd,
+		),
+		encoding.NewDecoder(),
+	)
+}
+
 // ToUTF8WithErr converts content to UTF8 encoding
 func ToUTF8WithErr(content []byte) (string, error) {
 	charsetLabel, err := DetectEncoding(content)
@@ -49,24 +78,8 @@ func ToUTF8WithErr(content []byte) (string, error) {
 
 // ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
 func ToUTF8WithFallback(content []byte) []byte {
-	charsetLabel, err := DetectEncoding(content)
-	if err != nil || charsetLabel == "UTF-8" {
-		return RemoveBOMIfPresent(content)
-	}
-
-	encoding, _ := charset.Lookup(charsetLabel)
-	if encoding == nil {
-		return content
-	}
-
-	// If there is an error, we concatenate the nicely decoded part and the
-	// original left over. This way we won't lose data.
-	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
-	if err != nil {
-		return append(result, content[n:]...)
-	}
-
-	return RemoveBOMIfPresent(result)
+	bs, _ := ioutil.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content)))
+	return bs
 }
 
 // ToUTF8 converts content to UTF8 encoding and ignore error

modules/csv/csv.go

@@ -7,7 +7,9 @@ package csv
 import (
 	"bytes"
 	"encoding/csv"
+	stdcsv "encoding/csv"
 	"errors"
+	"io"
 	"regexp"
 	"strings"
 
@@ -18,17 +20,31 @@ import (
 var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
 
 // CreateReader creates a csv.Reader with the given delimiter.
-func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader {
-	rd := csv.NewReader(bytes.NewReader(rawBytes))
+func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
+	rd := stdcsv.NewReader(input)
 	rd.Comma = delimiter
 	rd.TrimLeadingSpace = true
 	return rd
 }
 
 // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
-func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader {
-	delimiter := guessDelimiter(rawBytes)
-	return CreateReader(rawBytes, delimiter)
+func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) {
+	var data = make([]byte, 1e4)
+	size, err := rd.Read(data)
+	if err != nil {
+		return nil, err
+	}
+
+	delimiter := guessDelimiter(data[:size])
+
+	var newInput io.Reader
+	if size < 1e4 {
+		newInput = bytes.NewReader(data[:size])
+	} else {
+		newInput = io.MultiReader(bytes.NewReader(data), rd)
+	}
+
+	return CreateReader(newInput, delimiter), nil
 }
 
 // guessDelimiter scores the input CSV data against delimiters, and returns the best match.