remove goldmark meta and handle yaml frontmatter ourselves

Signed-off-by: Andrew Thornton <[email protected]>

Author: zeripath

go.mod

@@ -101,6 +101,7 @@ require (
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 	gopkg.in/ini.v1 v1.66.4
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	mvdan.cc/xurls/v2 v2.4.0
 	strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
 	xorm.io/builder v0.3.11
@@ -286,7 +287,6 @@ require (
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )
 

modules/markup/markdown/convertyaml.go

@@ -0,0 +1,84 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package markdown
+
+import (
+	"github.com/yuin/goldmark/ast"
+	east "github.com/yuin/goldmark/extension/ast"
+	"gopkg.in/yaml.v3"
+)
+
+func nodeToTable(meta *yaml.Node) ast.Node {
+	for {
+		if meta == nil {
+			return nil
+		}
+		switch meta.Kind {
+		case yaml.DocumentNode:
+			meta = meta.Content[0]
+			continue
+		default:
+		}
+		break
+	}
+	switch meta.Kind {
+	case yaml.MappingNode:
+		return mappingNodeToTable(meta)
+	case yaml.SequenceNode:
+		return sequenceNodeToTable(meta)
+	default:
+		return ast.NewString([]byte(meta.Value))
+	}
+}
+
+func mappingNodeToTable(meta *yaml.Node) ast.Node {
+	table := east.NewTable()
+	alignments := []east.Alignment{}
+	for i := 0; i < len(meta.Content); i += 2 {
+		alignments = append(alignments, east.AlignNone)
+	}
+
+	headerRow := east.NewTableRow(alignments)
+	valueRow := east.NewTableRow(alignments)
+	for i := 0; i < len(meta.Content); i += 2 {
+		cell := east.NewTableCell()
+
+		cell.AppendChild(cell, nodeToTable(meta.Content[i]))
+		headerRow.AppendChild(headerRow, cell)
+
+		if i+1 < len(meta.Content) {
+			cell = east.NewTableCell()
+			cell.AppendChild(cell, nodeToTable(meta.Content[i+1]))
+			valueRow.AppendChild(valueRow, cell)
+		}
+	}
+
+	table.AppendChild(table, east.NewTableHeader(headerRow))
+	table.AppendChild(table, valueRow)
+	return table
+}
+
+func sequenceNodeToTable(meta *yaml.Node) ast.Node {
+	table := east.NewTable()
+	alignments := []east.Alignment{east.AlignNone}
+	for _, item := range meta.Content {
+		row := east.NewTableRow(alignments)
+		cell := east.NewTableCell()
+		cell.AppendChild(cell, nodeToTable(item))
+		row.AppendChild(row, cell)
+		table.AppendChild(table, row)
+	}
+	return table
+}
+
+func nodeToDetails(meta *yaml.Node, icon string) ast.Node {
+	details := NewDetails()
+	summary := NewSummary()
+	summary.AppendChild(summary, NewIcon(icon))
+	details.AppendChild(details, summary)
+	details.AppendChild(details, nodeToTable(meta))
+
+	return details
+}

modules/markup/markdown/goldmark.go

@@ -15,7 +15,6 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	giteautil "code.gitea.io/gitea/modules/util"
 
-	meta "github.com/yuin/goldmark-meta"
 	"github.com/yuin/goldmark/ast"
 	east "github.com/yuin/goldmark/extension/ast"
 	"github.com/yuin/goldmark/parser"
@@ -32,20 +31,12 @@ type ASTTransformer struct{}
 
 // Transform transforms the given AST tree.
 func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
-	metaData := meta.GetItems(pc)
 	firstChild := node.FirstChild()
 	createTOC := false
 	ctx := pc.Get(renderContextKey).(*markup.RenderContext)
-	rc := &RenderConfig{
-		Meta: "table",
-		Icon: "table",
-		Lang: "",
-	}
-
-	if metaData != nil {
-		rc.ToRenderConfig(metaData)
-
-		metaNode := rc.toMetaNode(metaData)
+	rc := pc.Get(renderConfigKey).(*RenderConfig)
+	if rc.yamlNode != nil {
+		metaNode := rc.toMetaNode()
 		if metaNode != nil {
 			node.InsertBefore(node, firstChild, metaNode)
 		}

modules/markup/markdown/markdown.go

@@ -39,6 +39,7 @@ var (
 	isWikiKey        = parser.NewContextKey()
 	renderMetasKey   = parser.NewContextKey()
 	renderContextKey = parser.NewContextKey()
+	renderConfigKey  = parser.NewContextKey()
 )
 
 type limitWriter struct {
@@ -172,7 +173,18 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer)
 		log.Error("Unable to ReadAll: %v", err)
 		return err
 	}
-	if err := converter.Convert(giteautil.NormalizeEOL(buf), lw, parser.WithContext(pc)); err != nil {
+	buf = giteautil.NormalizeEOL(buf)
+
+	rc := &RenderConfig{
+		Meta: "table",
+		Icon: "table",
+		Lang: "",
+	}
+	buf, _ = ExtractMetadataBytes(buf, rc)
+
+	pc.Set(renderConfigKey, rc)
+
+	if err := converter.Convert(buf, lw, parser.WithContext(pc)); err != nil {
 		log.Error("Unable to render: %v", err)
 		return err
 	}

modules/markup/markdown/meta.go

@@ -5,47 +5,101 @@
 package markdown
 
 import (
+	"bytes"
 	"errors"
-	"strings"
+	"unicode"
+	"unicode/utf8"
 
-	"gopkg.in/yaml.v2"
+	"code.gitea.io/gitea/modules/log"
+	"gopkg.in/yaml.v3"
 )
 
-func isYAMLSeparator(line string) bool {
-	line = strings.TrimSpace(line)
-	for i := 0; i < len(line); i++ {
-		if line[i] != '-' {
+func isYAMLSeparator(line []byte) bool {
+	idx := 0
+	for ; idx < len(line); idx++ {
+		if line[idx] >= utf8.RuneSelf {
+			r, sz := utf8.DecodeRune(line[idx:])
+			if !unicode.IsSpace(r) {
+				return false
+			}
+			idx += sz
+			continue
+		}
+		if line[idx] != ' ' {
+			break
+		}
+	}
+	dashCount := 0
+	for ; idx < len(line); idx++ {
+		if line[idx] != '-' {
+			break
+		}
+		dashCount++
+	}
+	if dashCount < 3 {
+		return false
+	}
+	for ; idx < len(line); idx++ {
+		if line[idx] >= utf8.RuneSelf {
+			r, sz := utf8.DecodeRune(line[idx:])
+			if !unicode.IsSpace(r) {
+				return false
+			}
+			idx += sz
+			continue
+		}
+		if line[idx] != ' ' {
 			return false
 		}
 	}
-	return len(line) > 2
+	return true
 }
 
 // ExtractMetadata consumes a markdown file, parses YAML frontmatter,
 // and returns the frontmatter metadata separated from the markdown content
 func ExtractMetadata(contents string, out interface{}) (string, error) {
-	var front, body []string
-	lines := strings.Split(contents, "\n")
-	for idx, line := range lines {
-		if idx == 0 {
-			// First line has to be a separator
-			if !isYAMLSeparator(line) {
-				return "", errors.New("frontmatter must start with a separator line")
-			}
-			continue
+	body, err := ExtractMetadataBytes([]byte(contents), out)
+	return string(body), err
+}
+
+// ExtractMetadata consumes a markdown file, parses YAML frontmatter,
+// and returns the frontmatter metadata separated from the markdown content
+func ExtractMetadataBytes(contents []byte, out interface{}) ([]byte, error) {
+	var front, body []byte
+
+	start, end := 0, len(contents)
+	idx := bytes.IndexByte(contents[start:], '\n')
+	if idx >= 0 {
+		end = start + idx
+	}
+	line := contents[start:end]
+
+	if !isYAMLSeparator(line) {
+		return contents, errors.New("frontmatter must start with a separator line")
+	}
+	frontMatterStart := end + 1
+	for start = frontMatterStart; start < len(contents); start = end + 1 {
+		end = len(contents)
+		idx := bytes.IndexByte(contents[start:], '\n')
+		if idx >= 0 {
+			end = start + idx
 		}
+		line := contents[start:end]
 		if isYAMLSeparator(line) {
-			front, body = lines[1:idx], lines[idx+1:]
+			front = contents[frontMatterStart:start]
+			body = contents[end+1:]
 			break
 		}
 	}
 
 	if len(front) == 0 {
-		return "", errors.New("could not determine metadata")
+		return contents, errors.New("could not determine metadata")
 	}
 
-	if err := yaml.Unmarshal([]byte(strings.Join(front, "\n")), out); err != nil {
-		return "", err
+	log.Info("%s", string(front))
+
+	if err := yaml.Unmarshal(front, out); err != nil {
+		return contents, err
 	}
-	return strings.Join(body, "\n"), nil
+	return body, nil
 }

modules/markup/markdown/meta_test.go

@@ -45,6 +45,38 @@ func TestExtractMetadata(t *testing.T) {
 	})
 }
 
+func TestExtractMetadataBytes(t *testing.T) {
+	t.Run("ValidFrontAndBody", func(t *testing.T) {
+		var meta structs.IssueTemplate
+		body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest)), &meta)
+		assert.NoError(t, err)
+		assert.Equal(t, bodyTest, body)
+		assert.Equal(t, metaTest, meta)
+		assert.True(t, meta.Valid())
+	})
+
+	t.Run("NoFirstSeparator", func(t *testing.T) {
+		var meta structs.IssueTemplate
+		_, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", frontTest, sepTest, bodyTest)), &meta)
+		assert.Error(t, err)
+	})
+
+	t.Run("NoLastSeparator", func(t *testing.T) {
+		var meta structs.IssueTemplate
+		_, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, bodyTest)), &meta)
+		assert.Error(t, err)
+	})
+
+	t.Run("NoBody", func(t *testing.T) {
+		var meta structs.IssueTemplate
+		body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest)), &meta)
+		assert.NoError(t, err)
+		assert.Equal(t, "", body)
+		assert.Equal(t, metaTest, meta)
+		assert.True(t, meta.Valid())
+	})
+}
+
 var (
 	sepTest   = "-----"
 	frontTest = `name: Test