Remove newline when highlighting random chunks of code (#12180)

* Remove newline when highlighting random chunks of code

Somewhere when tokenizing a newline gets added to code formatted by chroma. This breaks the case of 'added-code' inside of an 'added-line' in a diff. Just remove any newline when processing chunks of code since we don't need it.

Fixes #12172

* don't process empty lines

* This is the proper way to fix this by telling chroma not to add the newline in the first place

Author: mrsdizzie

go.mod

@@ -100,8 +100,8 @@ require (
 	github.com/urfave/cli v1.20.0
 	github.com/xanzy/go-gitlab v0.31.0
 	github.com/yohcop/openid-go v1.0.0
-	github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691
 	github.com/yuin/goldmark v1.1.32
+	github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691
 	github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60
 	golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9
 	golang.org/x/net v0.0.0-20200602114024-627f9648deb9

modules/highlight/highlight.go

@@ -14,6 +14,7 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"github.com/alecthomas/chroma"
 	"github.com/alecthomas/chroma/formatters/html"
 	"github.com/alecthomas/chroma/lexers"
 	"github.com/alecthomas/chroma/styles"
@@ -68,7 +69,7 @@ func Code(fileName, code string) string {
 		lexer = lexers.Fallback
 	}
 
-	iterator, err := lexer.Tokenise(nil, string(code))
+	iterator, err := lexer.Tokenise(&chroma.TokeniseOptions{State: "root", Nested: true}, string(code))
 	if err != nil {
 		log.Error("Can't tokenize code: %v", err)
 		return code