Refactor renders (#15175)

* Refactor renders

* Some performance optimization

* Fix comment

* Transform reader

* Fix csv test

* Fix test

* Fix tests

* Improve optimaziation

* Fix test

* Fix test

* Detect file encoding with reader

* Improve optimaziation

* reduce memory usage

* improve code

* fix build

* Fix test

* Fix for go1.15

* Fix render

* Fix comment

* Fix lint

* Fix test

* Don't use NormalEOF when unnecessary

* revert change on util.go

* Apply suggestions from code review

Co-authored-by: zeripath <art27@cantab.net>

* rename function

* Take NormalEOF back

Co-authored-by: zeripath <art27@cantab.net>

contrib/pr/checkout.go

@@ -114,7 +114,7 @@ func runPR() {
 
 	log.Printf("[PR] Setting up router\n")
 	//routers.GlobalInit()
-	external.RegisterParsers()
+	external.RegisterRenderers()
 	markup.Init()
 	c := routes.NormalRoutes()
 

models/issue_comment.go

@@ -16,6 +16,7 @@ import (
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/references"
 	"code.gitea.io/gitea/modules/structs"
@@ -1178,8 +1179,13 @@ func findCodeComments(e Engine, opts FindCommentsOptions, issue *Issue, currentU
 			return nil, err
 		}
 
-		comment.RenderedContent = string(markdown.Render([]byte(comment.Content), issue.Repo.Link(),
-			issue.Repo.ComposeMetas()))
+		var err error
+		if comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+			URLPrefix: issue.Repo.Link(),
+			Metas:     issue.Repo.ComposeMetas(),
+		}, comment.Content); err != nil {
+			return nil, err
+		}
 	}
 	return comments[:n], nil
 }

models/repo.go

@@ -863,7 +863,10 @@ func (repo *Repository) getUsersWithAccessMode(e Engine, mode AccessMode) (_ []*
 
 // DescriptionHTML does special handles to description and return HTML string.
 func (repo *Repository) DescriptionHTML() template.HTML {
-	desc, err := markup.RenderDescriptionHTML([]byte(repo.Description), repo.HTMLURL(), repo.ComposeMetas())
+	desc, err := markup.RenderDescriptionHTML(&markup.RenderContext{
+		URLPrefix: repo.HTMLURL(),
+		Metas:     repo.ComposeMetas(),
+	}, repo.Description)
 	if err != nil {
 		log.Error("Failed to render description for %s (ID: %d): %v", repo.Name, repo.ID, err)
 		return template.HTML(markup.Sanitize(repo.Description))

models/repo_generate.go

@@ -5,13 +5,14 @@
 package models
 
 import (
+	"bufio"
+	"bytes"
 	"strconv"
 	"strings"
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/storage"
-	"code.gitea.io/gitea/modules/util"
 
 	"github.com/gobwas/glob"
 )
@@ -49,9 +50,9 @@ func (gt GiteaTemplate) Globs() []glob.Glob {
 	}
 
 	gt.globs = make([]glob.Glob, 0)
-	lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n")
-	for _, line := range lines {
-		line = strings.TrimSpace(line)
+	scanner := bufio.NewScanner(bytes.NewReader(gt.Content))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
 		if line == "" || strings.HasPrefix(line, "#") {
 			continue
 		}

modules/charset/charset.go

@@ -7,6 +7,8 @@ package charset
 import (
 	"bytes"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"strings"
 	"unicode/utf8"
 
@@ -21,6 +23,33 @@ import (
 // UTF8BOM is the utf-8 byte-order marker
 var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
 
+// ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible
+func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
+	var buf = make([]byte, 2048)
+	n, err := rd.Read(buf)
+	if err != nil {
+		return rd
+	}
+
+	charsetLabel, err := DetectEncoding(buf[:n])
+	if err != nil || charsetLabel == "UTF-8" {
+		return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return io.MultiReader(bytes.NewReader(buf[:n]), rd)
+	}
+
+	return transform.NewReader(
+		io.MultiReader(
+			bytes.NewReader(RemoveBOMIfPresent(buf[:n])),
+			rd,
+		),
+		encoding.NewDecoder(),
+	)
+}
+
 // ToUTF8WithErr converts content to UTF8 encoding
 func ToUTF8WithErr(content []byte) (string, error) {
 	charsetLabel, err := DetectEncoding(content)
@@ -49,24 +78,8 @@ func ToUTF8WithErr(content []byte) (string, error) {
 
 // ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
 func ToUTF8WithFallback(content []byte) []byte {
-	charsetLabel, err := DetectEncoding(content)
-	if err != nil || charsetLabel == "UTF-8" {
-		return RemoveBOMIfPresent(content)
-	}
-
-	encoding, _ := charset.Lookup(charsetLabel)
-	if encoding == nil {
-		return content
-	}
-
-	// If there is an error, we concatenate the nicely decoded part and the
-	// original left over. This way we won't lose data.
-	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
-	if err != nil {
-		return append(result, content[n:]...)
-	}
-
-	return RemoveBOMIfPresent(result)
+	bs, _ := ioutil.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content)))
+	return bs
 }
 
 // ToUTF8 converts content to UTF8 encoding and ignore error

modules/csv/csv.go

@@ -7,7 +7,9 @@ package csv
 import (
 	"bytes"
 	"encoding/csv"
+	stdcsv "encoding/csv"
 	"errors"
+	"io"
 	"regexp"
 	"strings"
 
@@ -18,17 +20,31 @@ import (
 var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
 
 // CreateReader creates a csv.Reader with the given delimiter.
-func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader {
-	rd := csv.NewReader(bytes.NewReader(rawBytes))
+func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
+	rd := stdcsv.NewReader(input)
 	rd.Comma = delimiter
 	rd.TrimLeadingSpace = true
 	return rd
 }
 
 // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
-func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader {
-	delimiter := guessDelimiter(rawBytes)
-	return CreateReader(rawBytes, delimiter)
+func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) {
+	var data = make([]byte, 1e4)
+	size, err := rd.Read(data)
+	if err != nil {
+		return nil, err
+	}
+
+	delimiter := guessDelimiter(data[:size])
+
+	var newInput io.Reader
+	if size < 1e4 {
+		newInput = bytes.NewReader(data[:size])
+	} else {
+		newInput = io.MultiReader(bytes.NewReader(data), rd)
+	}
+
+	return CreateReader(newInput, delimiter), nil
 }
 
 // guessDelimiter scores the input CSV data against delimiters, and returns the best match.

modules/csv/csv_test.go

@@ -5,20 +5,23 @@
 package csv
 
 import (
+	"bytes"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func TestCreateReader(t *testing.T) {
-	rd := CreateReader([]byte{}, ',')
+	rd := CreateReader(bytes.NewReader([]byte{}), ',')
 	assert.Equal(t, ',', rd.Comma)
 }
 
 func TestCreateReaderAndGuessDelimiter(t *testing.T) {
 	input := "a;b;c\n1;2;3\n4;5;6"
 
-	rd := CreateReaderAndGuessDelimiter([]byte(input))
+	rd, err := CreateReaderAndGuessDelimiter(strings.NewReader(input))
+	assert.NoError(t, err)
 	assert.Equal(t, ';', rd.Comma)
 }
 

modules/markup/csv/csv.go

@@ -5,9 +5,11 @@
 package markup
 
 import (
+	"bufio"
 	"bytes"
 	"html"
 	"io"
+	"io/ioutil"
 	"strconv"
 
 	"code.gitea.io/gitea/modules/csv"
@@ -16,55 +18,89 @@ import (
 )
 
 func init() {
-	markup.RegisterParser(Parser{})
+	markup.RegisterRenderer(Renderer{})
 }
 
-// Parser implements markup.Parser for csv files
-type Parser struct {
+// Renderer implements markup.Renderer for csv files
+type Renderer struct {
 }
 
-// Name implements markup.Parser
-func (Parser) Name() string {
+// Name implements markup.Renderer
+func (Renderer) Name() string {
 	return "csv"
 }
 
-// NeedPostProcess implements markup.Parser
-func (Parser) NeedPostProcess() bool { return false }
+// NeedPostProcess implements markup.Renderer
+func (Renderer) NeedPostProcess() bool { return false }
 
-// Extensions implements markup.Parser
-func (Parser) Extensions() []string {
+// Extensions implements markup.Renderer
+func (Renderer) Extensions() []string {
 	return []string{".csv", ".tsv"}
 }
 
-// Render implements markup.Parser
-func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	var tmpBlock bytes.Buffer
+func writeField(w io.Writer, element, class, field string) error {
+	if _, err := io.WriteString(w, "<"); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, element); err != nil {
+		return err
+	}
+	if len(class) > 0 {
+		if _, err := io.WriteString(w, " class=\""); err != nil {
+			return err
+		}
+		if _, err := io.WriteString(w, class); err != nil {
+			return err
+		}
+		if _, err := io.WriteString(w, "\""); err != nil {
+			return err
+		}
+	}
+	if _, err := io.WriteString(w, ">"); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, html.EscapeString(field)); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, "</"); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, element); err != nil {
+		return err
+	}
+	_, err := io.WriteString(w, ">")
+	return err
+}
+
+// Render implements markup.Renderer
+func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
+	var tmpBlock = bufio.NewWriter(output)
+
+	// FIXME: don't read all to memory
+	rawBytes, err := ioutil.ReadAll(input)
+	if err != nil {
+		return err
+	}
 
 	if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) {
-		tmpBlock.WriteString("<pre>")
-		tmpBlock.WriteString(html.EscapeString(string(rawBytes)))
-		tmpBlock.WriteString("</pre>")
-		return tmpBlock.Bytes()
+		if _, err := tmpBlock.WriteString("<pre>"); err != nil {
+			return err
+		}
+		if _, err := tmpBlock.WriteString(html.EscapeString(string(rawBytes))); err != nil {
+			return err
+		}
+		_, err = tmpBlock.WriteString("</pre>")
+		return err
 	}
 
-	rd := csv.CreateReaderAndGuessDelimiter(rawBytes)
-
-	writeField := func(element, class, field string) {
-		tmpBlock.WriteString("<")
-		tmpBlock.WriteString(element)
-		if len(class) > 0 {
-			tmpBlock.WriteString(" class=\"")
-			tmpBlock.WriteString(class)
-			tmpBlock.WriteString("\"")
-		}
-		tmpBlock.WriteString(">")
-		tmpBlock.WriteString(html.EscapeString(field))
-		tmpBlock.WriteString("</")
-		tmpBlock.WriteString(element)
-		tmpBlock.WriteString(">")
+	rd, err := csv.CreateReaderAndGuessDelimiter(bytes.NewReader(rawBytes))
+	if err != nil {
+		return err
 	}
 
-	tmpBlock.WriteString(`<table class="data-table">`)
+	if _, err := tmpBlock.WriteString(`<table class="data-table">`); err != nil {
+		return err
+	}
 	row := 1
 	for {
 		fields, err := rd.Read()
@@ -74,20 +110,29 @@ func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string,
 		if err != nil {
 			continue
 		}
-		tmpBlock.WriteString("<tr>")
+		if _, err := tmpBlock.WriteString("<tr>"); err != nil {
+			return err
+		}
 		element := "td"
 		if row == 1 {
 			element = "th"
 		}
-		writeField(element, "line-num", strconv.Itoa(row))
-		for _, field := range fields {
-			writeField(element, "", field)
+		if err := writeField(tmpBlock, element, "line-num", strconv.Itoa(row)); err != nil {
+			return err
+		}
+		for _, field := range fields {
+			if err := writeField(tmpBlock, element, "", field); err != nil {
+				return err
+			}
+		}
+		if _, err := tmpBlock.WriteString("</tr>"); err != nil {
+			return err
 		}
-		tmpBlock.WriteString("</tr>")
 
 		row++
 	}
-	tmpBlock.WriteString("</table>")
-
-	return tmpBlock.Bytes()
+	if _, err = tmpBlock.WriteString("</table>"); err != nil {
+		return err
+	}
+	return tmpBlock.Flush()
 }

modules/markup/csv/csv_test.go

@@ -5,13 +5,16 @@
 package markup
 
 import (
+	"strings"
 	"testing"
 
+	"code.gitea.io/gitea/modules/markup"
+
 	"github.com/stretchr/testify/assert"
 )
 
 func TestRenderCSV(t *testing.T) {
-	var parser Parser
+	var render Renderer
 	var kases = map[string]string{
 		"a":        "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>a</th></tr></table>",
 		"1,2":      "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>1</th><th>2</th></tr></table>",
@@ -20,7 +23,9 @@ func TestRenderCSV(t *testing.T) {
 	}
 
 	for k, v := range kases {
-		res := parser.Render([]byte(k), "", nil, false)
-		assert.EqualValues(t, v, string(res))
+		var buf strings.Builder
+		err := render.Render(&markup.RenderContext{}, strings.NewReader(k), &buf)
+		assert.NoError(t, err)
+		assert.EqualValues(t, v, buf.String())
 	}
 }

modules/markup/external/external.go

@@ -5,7 +5,7 @@
 package external
 
 import (
-	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -19,32 +19,32 @@ import (
 	"code.gitea.io/gitea/modules/util"
 )
 
-// RegisterParsers registers all supported third part parsers according settings
-func RegisterParsers() {
-	for _, parser := range setting.ExternalMarkupParsers {
-		if parser.Enabled && parser.Command != "" && len(parser.FileExtensions) > 0 {
-			markup.RegisterParser(&Parser{parser})
+// RegisterRenderers registers all supported third part renderers according settings
+func RegisterRenderers() {
+	for _, renderer := range setting.ExternalMarkupRenderers {
+		if renderer.Enabled && renderer.Command != "" && len(renderer.FileExtensions) > 0 {
+			markup.RegisterRenderer(&Renderer{renderer})
 		}
 	}
 }
 
-// Parser implements markup.Parser for external tools
-type Parser struct {
-	setting.MarkupParser
+// Renderer implements markup.Renderer for external tools
+type Renderer struct {
+	setting.MarkupRenderer
 }
 
 // Name returns the external tool name
-func (p *Parser) Name() string {
+func (p *Renderer) Name() string {
 	return p.MarkupName
 }
 
-// NeedPostProcess implements markup.Parser
-func (p *Parser) NeedPostProcess() bool {
-	return p.MarkupParser.NeedPostProcess
+// NeedPostProcess implements markup.Renderer
+func (p *Renderer) NeedPostProcess() bool {
+	return p.MarkupRenderer.NeedPostProcess
 }
 
 // Extensions returns the supported extensions of the tool
-func (p *Parser) Extensions() []string {
+func (p *Renderer) Extensions() []string {
 	return p.FileExtensions
 }
 
@@ -56,14 +56,10 @@ func envMark(envName string) string {
 }
 
 // Render renders the data of the document to HTML via the external tool.
-func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
+func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
 	var (
-		bs           []byte
-		buf          = bytes.NewBuffer(bs)
-		rd           = bytes.NewReader(rawBytes)
-		urlRawPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1)
-
-		command = strings.NewReplacer(envMark("GITEA_PREFIX_SRC"), urlPrefix,
+		urlRawPrefix = strings.Replace(ctx.URLPrefix, "/src/", "/raw/", 1)
+		command      = strings.NewReplacer(envMark("GITEA_PREFIX_SRC"), ctx.URLPrefix,
 			envMark("GITEA_PREFIX_RAW"), urlRawPrefix).Replace(p.Command)
 		commands = strings.Fields(command)
 		args     = commands[1:]
@@ -73,8 +69,7 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri
 		// write to temp file
 		f, err := ioutil.TempFile("", "gitea_input")
 		if err != nil {
-			log.Error("%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err)
-			return []byte("")
+			return fmt.Errorf("%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err)
 		}
 		tmpPath := f.Name()
 		defer func() {
@@ -83,17 +78,15 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri
 			}
 		}()
 
-		_, err = io.Copy(f, rd)
+		_, err = io.Copy(f, input)
 		if err != nil {
 			f.Close()
-			log.Error("%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err)
-			return []byte("")
+			return fmt.Errorf("%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err)
 		}
 
 		err = f.Close()
 		if err != nil {
-			log.Error("%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err)
-			return []byte("")
+			return fmt.Errorf("%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err)
 		}
 		args = append(args, f.Name())
 	}
@@ -101,16 +94,15 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri
 	cmd := exec.Command(commands[0], args...)
 	cmd.Env = append(
 		os.Environ(),
-		"GITEA_PREFIX_SRC="+urlPrefix,
+		"GITEA_PREFIX_SRC="+ctx.URLPrefix,
 		"GITEA_PREFIX_RAW="+urlRawPrefix,
 	)
 	if !p.IsInputFile {
-		cmd.Stdin = rd
+		cmd.Stdin = input
 	}
-	cmd.Stdout = buf
+	cmd.Stdout = output
 	if err := cmd.Run(); err != nil {
-		log.Error("%s render run command %s %v failed: %v", p.Name(), commands[0], args, err)
-		return []byte("")
+		return fmt.Errorf("%s render run command %s %v failed: %v", p.Name(), commands[0], args, err)
 	}
-	return buf.Bytes()
+	return nil
 }

modules/markup/html_internal_test.go

@@ -61,8 +61,8 @@ var localMetas = map[string]string{
 func TestRender_IssueIndexPattern(t *testing.T) {
 	// numeric: render inputs without valid mentions
 	test := func(s string) {
-		testRenderIssueIndexPattern(t, s, s, nil)
-		testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: numericMetas})
+		testRenderIssueIndexPattern(t, s, s, &RenderContext{})
+		testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: numericMetas})
 	}
 
 	// should not render anything when there are no mentions
@@ -109,13 +109,13 @@ func TestRender_IssueIndexPattern2(t *testing.T) {
 			links[i] = numericIssueLink(util.URLJoin(setting.AppSubURL, path), "ref-issue", index, marker)
 		}
 		expectedNil := fmt.Sprintf(expectedFmt, links...)
-		testRenderIssueIndexPattern(t, s, expectedNil, &postProcessCtx{metas: localMetas})
+		testRenderIssueIndexPattern(t, s, expectedNil, &RenderContext{Metas: localMetas})
 
 		for i, index := range indices {
 			links[i] = numericIssueLink(prefix, "ref-issue", index, marker)
 		}
 		expectedNum := fmt.Sprintf(expectedFmt, links...)
-		testRenderIssueIndexPattern(t, s, expectedNum, &postProcessCtx{metas: numericMetas})
+		testRenderIssueIndexPattern(t, s, expectedNum, &RenderContext{Metas: numericMetas})
 	}
 
 	// should render freestanding mentions
@@ -150,7 +150,7 @@ func TestRender_IssueIndexPattern3(t *testing.T) {
 
 	// alphanumeric: render inputs without valid mentions
 	test := func(s string) {
-		testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: alphanumericMetas})
+		testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: alphanumericMetas})
 	}
 	test("")
 	test("this is a test")
@@ -181,25 +181,22 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
 			links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue", name)
 		}
 		expected := fmt.Sprintf(expectedFmt, links...)
-		testRenderIssueIndexPattern(t, s, expected, &postProcessCtx{metas: alphanumericMetas})
+		testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas})
 	}
 	test("OTT-1234 test", "%s test", "OTT-1234")
 	test("test T-12 issue", "test %s issue", "T-12")
 	test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890")
 }
 
-func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *postProcessCtx) {
-	if ctx == nil {
-		ctx = new(postProcessCtx)
-	}
-	ctx.procs = []processor{issueIndexPatternProcessor}
-	if ctx.urlPrefix == "" {
-		ctx.urlPrefix = AppSubURL
+func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) {
+	if ctx.URLPrefix == "" {
+		ctx.URLPrefix = AppSubURL
 	}
 
-	res, err := ctx.postProcess([]byte(input))
+	var buf strings.Builder
+	err := postProcess(ctx, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf)
 	assert.NoError(t, err)
-	assert.Equal(t, expected, string(res))
+	assert.Equal(t, expected, buf.String())
 }
 
 func TestRender_AutoLink(t *testing.T) {
@@ -207,12 +204,22 @@ func TestRender_AutoLink(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer, err := PostProcess([]byte(input), setting.AppSubURL, localMetas, false)
+		var buffer strings.Builder
+		err := PostProcess(&RenderContext{
+			URLPrefix: setting.AppSubURL,
+			Metas:     localMetas,
+		}, strings.NewReader(input), &buffer)
 		assert.Equal(t, err, nil)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
-		buffer, err = PostProcess([]byte(input), setting.AppSubURL, localMetas, true)
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String()))
+
+		buffer.Reset()
+		err = PostProcess(&RenderContext{
+			URLPrefix: setting.AppSubURL,
+			Metas:     localMetas,
+			IsWiki:    true,
+		}, strings.NewReader(input), &buffer)
 		assert.Equal(t, err, nil)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String()))
 	}
 
 	// render valid issue URLs
@@ -235,15 +242,13 @@ func TestRender_FullIssueURLs(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		ctx := new(postProcessCtx)
-		ctx.procs = []processor{fullIssuePatternProcessor}
-		if ctx.urlPrefix == "" {
-			ctx.urlPrefix = AppSubURL
-		}
-		ctx.metas = localMetas
-		result, err := ctx.postProcess([]byte(input))
+		var result strings.Builder
+		err := postProcess(&RenderContext{
+			URLPrefix: AppSubURL,
+			Metas:     localMetas,
+		}, []processor{fullIssuePatternProcessor}, strings.NewReader(input), &result)
 		assert.NoError(t, err)
-		assert.Equal(t, expected, string(result))
+		assert.Equal(t, expected, result.String())
 	}
 	test("Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6",
 		"Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6")

modules/markup/html_test.go

@@ -28,7 +28,12 @@ func TestRender_Commits(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString(".md", input, setting.AppSubURL, localMetas)
+		buffer, err := RenderString(&RenderContext{
+			Filename:  ".md",
+			URLPrefix: setting.AppSubURL,
+			Metas:     localMetas,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
@@ -59,7 +64,12 @@ func TestRender_CrossReferences(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString("a.md", input, setting.AppSubURL, localMetas)
+		buffer, err := RenderString(&RenderContext{
+			Filename:  "a.md",
+			URLPrefix: setting.AppSubURL,
+			Metas:     localMetas,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
@@ -91,7 +101,11 @@ func TestRender_links(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString("a.md", input, setting.AppSubURL, nil)
+		buffer, err := RenderString(&RenderContext{
+			Filename:  "a.md",
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 	// Text that should be turned into URL
@@ -187,8 +201,12 @@ func TestRender_email(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString("a.md", input, setting.AppSubURL, nil)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
+		res, err := RenderString(&RenderContext{
+			Filename:  "a.md",
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res))
 	}
 	// Text that should be turned into email link
 
@@ -242,7 +260,11 @@ func TestRender_emoji(t *testing.T) {
 
 	test := func(input, expected string) {
 		expected = strings.ReplaceAll(expected, "&", "&")
-		buffer := RenderString("a.md", input, setting.AppSubURL, nil)
+		buffer, err := RenderString(&RenderContext{
+			Filename:  "a.md",
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
@@ -291,9 +313,17 @@ func TestRender_ShortLinks(t *testing.T) {
 	tree := util.URLJoin(AppSubURL, "src", "master")
 
 	test := func(input, expected, expectedWiki string) {
-		buffer := markdown.RenderString(input, tree, nil)
+		buffer, err := markdown.RenderString(&RenderContext{
+			URLPrefix: tree,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
-		buffer = markdown.RenderWiki([]byte(input), setting.AppSubURL, localMetas)
+		buffer, err = markdown.RenderString(&RenderContext{
+			URLPrefix: setting.AppSubURL,
+			Metas:     localMetas,
+			IsWiki:    true,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(buffer))
 	}
 
@@ -395,16 +425,22 @@ func Test_ParseClusterFuzz(t *testing.T) {
 
 	data := "<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY "
 
-	val, err := PostProcess([]byte(data), "https://example.com", localMetas, false)
-
+	var res strings.Builder
+	err := PostProcess(&RenderContext{
+		URLPrefix: "https://example.com",
+		Metas:     localMetas,
+	}, strings.NewReader(data), &res)
 	assert.NoError(t, err)
-	assert.NotContains(t, string(val), "<html")
+	assert.NotContains(t, res.String(), "<html")
 
 	data = "<!DOCTYPE html>\n<A><maTH><tr><MN><bodY ÿ><temPlate></template><tH><tr></A><tH><d<bodY "
 
-	val, err = PostProcess([]byte(data), "https://example.com", localMetas, false)
+	res.Reset()
+	err = PostProcess(&RenderContext{
+		URLPrefix: "https://example.com",
+		Metas:     localMetas,
+	}, strings.NewReader(data), &res)
 
 	assert.NoError(t, err)
-
-	assert.NotContains(t, string(val), "<html")
+	assert.NotContains(t, res.String(), "<html")
 }

modules/markup/markdown/markdown.go

@@ -8,6 +8,7 @@ package markdown
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"strings"
 	"sync"
 
@@ -73,17 +74,17 @@ func (l *limitWriter) CloseWithError(err error) error {
 	return l.w.CloseWithError(err)
 }
 
-// NewGiteaParseContext creates a parser.Context with the gitea context set
-func NewGiteaParseContext(urlPrefix string, metas map[string]string, isWiki bool) parser.Context {
+// newParserContext creates a parser.Context with the render context set
+func newParserContext(ctx *markup.RenderContext) parser.Context {
 	pc := parser.NewContext(parser.WithIDs(newPrefixedIDs()))
-	pc.Set(urlPrefixKey, urlPrefix)
-	pc.Set(isWikiKey, isWiki)
-	pc.Set(renderMetasKey, metas)
+	pc.Set(urlPrefixKey, ctx.URLPrefix)
+	pc.Set(isWikiKey, ctx.IsWiki)
+	pc.Set(renderMetasKey, ctx.Metas)
 	return pc
 }
 
 // actualRender renders Markdown to HTML without handling special links.
-func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown bool) []byte {
+func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
 	once.Do(func() {
 		converter = goldmark.New(
 			goldmark.WithExtensions(extension.Table,
@@ -169,7 +170,7 @@ func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMa
 		limit: setting.UI.MaxDisplayFileSize * 3,
 	}
 
-	// FIXME: should we include a timeout that closes the pipe to abort the parser and sanitizer if it takes too long?
+	// FIXME: should we include a timeout that closes the pipe to abort the renderer and sanitizer if it takes too long?
 	go func() {
 		defer func() {
 			err := recover()
@@ -184,18 +185,26 @@ func actualRender(body []byte, urlPrefix string, metas map[string]string, wikiMa
 			_ = lw.CloseWithError(fmt.Errorf("%v", err))
 		}()
 
-		pc := NewGiteaParseContext(urlPrefix, metas, wikiMarkdown)
-		if err := converter.Convert(giteautil.NormalizeEOL(body), lw, parser.WithContext(pc)); err != nil {
+		// FIXME: Don't read all to memory, but goldmark doesn't support
+		pc := newParserContext(ctx)
+		buf, err := ioutil.ReadAll(input)
+		if err != nil {
+			log.Error("Unable to ReadAll: %v", err)
+			return
+		}
+		if err := converter.Convert(giteautil.NormalizeEOL(buf), lw, parser.WithContext(pc)); err != nil {
 			log.Error("Unable to render: %v", err)
 			_ = lw.CloseWithError(err)
 			return
 		}
 		_ = lw.Close()
 	}()
-	return markup.SanitizeReader(rd).Bytes()
+	buf := markup.SanitizeReader(rd)
+	_, err := io.Copy(output, buf)
+	return err
 }
 
-func render(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown bool) (ret []byte) {
+func render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
 	defer func() {
 		err := recover()
 		if err == nil {
@@ -206,9 +215,13 @@ func render(body []byte, urlPrefix string, metas map[string]string, wikiMarkdown
 		if log.IsDebug() {
 			log.Debug("Panic in markdown: %v\n%s", err, string(log.Stack(2)))
 		}
-		ret = markup.SanitizeBytes(body)
+		ret := markup.SanitizeReader(input)
+		_, err = io.Copy(output, ret)
+		if err != nil {
+			log.Error("SanitizeReader failed: %v", err)
+		}
 	}()
-	return actualRender(body, urlPrefix, metas, wikiMarkdown)
+	return actualRender(ctx, input, output)
 }
 
 var (
@@ -217,48 +230,59 @@ var (
 )
 
 func init() {
-	markup.RegisterParser(Parser{})
+	markup.RegisterRenderer(Renderer{})
 }
 
-// Parser implements markup.Parser
-type Parser struct{}
+// Renderer implements markup.Renderer
+type Renderer struct{}
 
-// Name implements markup.Parser
-func (Parser) Name() string {
+// Name implements markup.Renderer
+func (Renderer) Name() string {
 	return MarkupName
 }
 
-// NeedPostProcess implements markup.Parser
-func (Parser) NeedPostProcess() bool { return true }
+// NeedPostProcess implements markup.Renderer
+func (Renderer) NeedPostProcess() bool { return true }
 
-// Extensions implements markup.Parser
-func (Parser) Extensions() []string {
+// Extensions implements markup.Renderer
+func (Renderer) Extensions() []string {
 	return setting.Markdown.FileExtensions
 }
 
-// Render implements markup.Parser
-func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	return render(rawBytes, urlPrefix, metas, isWiki)
+// Render implements markup.Renderer
+func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
+	return render(ctx, input, output)
 }
 
 // Render renders Markdown to HTML with all specific handling stuff.
-func Render(rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
-	return markup.Render("a.md", rawBytes, urlPrefix, metas)
+func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
+	if ctx.Filename == "" {
+		ctx.Filename = "a.md"
+	}
+	return markup.Render(ctx, input, output)
+}
+
+// RenderString renders Markdown string to HTML with all specific handling stuff and return string
+func RenderString(ctx *markup.RenderContext, content string) (string, error) {
+	var buf strings.Builder
+	if err := Render(ctx, strings.NewReader(content), &buf); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
 }
 
 // RenderRaw renders Markdown to HTML without handling special links.
-func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte {
-	return render(body, urlPrefix, map[string]string{}, wikiMarkdown)
+func RenderRaw(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
+	return render(ctx, input, output)
 }
 
-// RenderString renders Markdown to HTML with special links and returns string type.
-func RenderString(raw, urlPrefix string, metas map[string]string) string {
-	return markup.RenderString("a.md", raw, urlPrefix, metas)
+// RenderRawString renders Markdown to HTML without handling special links and return string
+func RenderRawString(ctx *markup.RenderContext, content string) (string, error) {
+	var buf strings.Builder
+	if err := RenderRaw(ctx, strings.NewReader(content), &buf); err != nil {
+		return "", err
 	}
-
-// RenderWiki renders markdown wiki page to HTML and return HTML string
-func RenderWiki(rawBytes []byte, urlPrefix string, metas map[string]string) string {
-	return markup.RenderWiki("a.md", rawBytes, urlPrefix, metas)
+	return buf.String(), nil
 }
 
 // IsMarkdownFile reports whether name looks like a Markdown file

modules/markup/markdown/markdown_test.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"testing"
 
+	"code.gitea.io/gitea/modules/markup"
 	. "code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
@@ -31,10 +32,17 @@ func TestRender_StandardLinks(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected, expectedWiki string) {
-		buffer := RenderString(input, setting.AppSubURL, nil)
+		buffer, err := RenderString(&markup.RenderContext{
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
-		bufferWiki := RenderWiki([]byte(input), setting.AppSubURL, nil)
-		assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(bufferWiki))
+
+		buffer, err = RenderString(&markup.RenderContext{
+			URLPrefix: setting.AppSubURL,
+			IsWiki:    true,
+		}, input)
+		assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(buffer))
 	}
 
 	googleRendered := `<p><a href="https://google.com/" rel="nofollow">https://google.com/</a></p>`
@@ -74,7 +82,10 @@ func TestRender_Images(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString(input, setting.AppSubURL, nil)
+		buffer, err := RenderString(&markup.RenderContext{
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
@@ -261,7 +272,12 @@ func TestTotal_RenderWiki(t *testing.T) {
 	answers := testAnswers(util.URLJoin(AppSubURL, "wiki/"), util.URLJoin(AppSubURL, "wiki", "raw/"))
 
 	for i := 0; i < len(sameCases); i++ {
-		line := RenderWiki([]byte(sameCases[i]), AppSubURL, localMetas)
+		line, err := RenderString(&markup.RenderContext{
+			URLPrefix: AppSubURL,
+			Metas:     localMetas,
+			IsWiki:    true,
+		}, sameCases[i])
+		assert.NoError(t, err)
 		assert.Equal(t, answers[i], line)
 	}
 
@@ -279,7 +295,11 @@ func TestTotal_RenderWiki(t *testing.T) {
 	}
 
 	for i := 0; i < len(testCases); i += 2 {
-		line := RenderWiki([]byte(testCases[i]), AppSubURL, nil)
+		line, err := RenderString(&markup.RenderContext{
+			URLPrefix: AppSubURL,
+			IsWiki:    true,
+		}, testCases[i])
+		assert.NoError(t, err)
 		assert.Equal(t, testCases[i+1], line)
 	}
 }
@@ -288,31 +308,40 @@ func TestTotal_RenderString(t *testing.T) {
 	answers := testAnswers(util.URLJoin(AppSubURL, "src", "master/"), util.URLJoin(AppSubURL, "raw", "master/"))
 
 	for i := 0; i < len(sameCases); i++ {
-		line := RenderString(sameCases[i], util.URLJoin(AppSubURL, "src", "master/"), localMetas)
+		line, err := RenderString(&markup.RenderContext{
+			URLPrefix: util.URLJoin(AppSubURL, "src", "master/"),
+			Metas:     localMetas,
+		}, sameCases[i])
+		assert.NoError(t, err)
 		assert.Equal(t, answers[i], line)
 	}
 
 	testCases := []string{}
 
 	for i := 0; i < len(testCases); i += 2 {
-		line := RenderString(testCases[i], AppSubURL, nil)
+		line, err := RenderString(&markup.RenderContext{
+			URLPrefix: AppSubURL,
+		}, testCases[i])
+		assert.NoError(t, err)
 		assert.Equal(t, testCases[i+1], line)
 	}
 }
 
 func TestRender_RenderParagraphs(t *testing.T) {
 	test := func(t *testing.T, str string, cnt int) {
-		unix := []byte(str)
-		res := string(RenderRaw(unix, "", false))
-		assert.Equal(t, strings.Count(res, "<p"), cnt, "Rendered result for unix should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
+		res, err := RenderRawString(&markup.RenderContext{}, str)
+		assert.NoError(t, err)
+		assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for unix should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
 
-		mac := []byte(strings.ReplaceAll(str, "\n", "\r"))
-		res = string(RenderRaw(mac, "", false))
-		assert.Equal(t, strings.Count(res, "<p"), cnt, "Rendered result for mac should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
+		mac := strings.ReplaceAll(str, "\n", "\r")
+		res, err = RenderRawString(&markup.RenderContext{}, mac)
+		assert.NoError(t, err)
+		assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for mac should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
 
-		dos := []byte(strings.ReplaceAll(str, "\n", "\r\n"))
-		res = string(RenderRaw(dos, "", false))
-		assert.Equal(t, strings.Count(res, "<p"), cnt, "Rendered result for windows should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
+		dos := strings.ReplaceAll(str, "\n", "\r\n")
+		res, err = RenderRawString(&markup.RenderContext{}, dos)
+		assert.NoError(t, err)
+		assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for windows should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
 	}
 
 	test(t, "\nOne\nTwo\nThree", 1)
@@ -337,7 +366,8 @@ func TestMarkdownRenderRaw(t *testing.T) {
 	}
 
 	for _, testcase := range testcases {
-		_ = RenderRaw(testcase, "", false)
+		_, err := RenderRawString(&markup.RenderContext{}, string(testcase))
+		assert.NoError(t, err)
 	}
 }
 
@@ -348,7 +378,8 @@ func TestRenderSiblingImages_Issue12925(t *testing.T) {
 	expected := `<p><a href="/image1" rel="nofollow"><img src="/image1" alt="image1"></a><br>
 <a href="/image2" rel="nofollow"><img src="/image2" alt="image2"></a></p>
 `
-	res := string(RenderRaw([]byte(testcase), "", false))
+	res, err := RenderRawString(&markup.RenderContext{}, testcase)
+	assert.NoError(t, err)
 	assert.Equal(t, expected, res)
 
 }

modules/markup/markup.go

@@ -1,143 +0,0 @@
-// Copyright 2017 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package markup
-
-import (
-	"path/filepath"
-	"strings"
-
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
-)
-
-// Init initialize regexps for markdown parsing
-func Init() {
-	getIssueFullPattern()
-	NewSanitizer()
-	if len(setting.Markdown.CustomURLSchemes) > 0 {
-		CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes)
-	}
-
-	// since setting maybe changed extensions, this will reload all parser extensions mapping
-	extParsers = make(map[string]Parser)
-	for _, parser := range parsers {
-		for _, ext := range parser.Extensions() {
-			extParsers[strings.ToLower(ext)] = parser
-		}
-	}
-}
-
-// Parser defines an interface for parsering markup file to HTML
-type Parser interface {
-	Name() string // markup format name
-	Extensions() []string
-	NeedPostProcess() bool
-	Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte
-}
-
-var (
-	extParsers = make(map[string]Parser)
-	parsers    = make(map[string]Parser)
-)
-
-// RegisterParser registers a new markup file parser
-func RegisterParser(parser Parser) {
-	parsers[parser.Name()] = parser
-	for _, ext := range parser.Extensions() {
-		extParsers[strings.ToLower(ext)] = parser
-	}
-}
-
-// GetParserByFileName get parser by filename
-func GetParserByFileName(filename string) Parser {
-	extension := strings.ToLower(filepath.Ext(filename))
-	return extParsers[extension]
-}
-
-// GetParserByType returns a parser according type
-func GetParserByType(tp string) Parser {
-	return parsers[tp]
-}
-
-// Render renders markup file to HTML with all specific handling stuff.
-func Render(filename string, rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
-	return renderFile(filename, rawBytes, urlPrefix, metas, false)
-}
-
-// RenderByType renders markup to HTML with special links and returns string type.
-func RenderByType(tp string, rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
-	return renderByType(tp, rawBytes, urlPrefix, metas, false)
-}
-
-// RenderString renders Markdown to HTML with special links and returns string type.
-func RenderString(filename string, raw, urlPrefix string, metas map[string]string) string {
-	return string(renderFile(filename, []byte(raw), urlPrefix, metas, false))
-}
-
-// RenderWiki renders markdown wiki page to HTML and return HTML string
-func RenderWiki(filename string, rawBytes []byte, urlPrefix string, metas map[string]string) string {
-	return string(renderFile(filename, rawBytes, urlPrefix, metas, true))
-}
-
-func render(parser Parser, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	result := parser.Render(rawBytes, urlPrefix, metas, isWiki)
-	if parser.NeedPostProcess() {
-		var err error
-		// TODO: one day the error should be returned.
-		result, err = PostProcess(result, urlPrefix, metas, isWiki)
-		if err != nil {
-			log.Error("PostProcess: %v", err)
-		}
-	}
-	return SanitizeBytes(result)
-}
-
-func renderByType(tp string, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	if parser, ok := parsers[tp]; ok {
-		return render(parser, rawBytes, urlPrefix, metas, isWiki)
-	}
-	return nil
-}
-
-func renderFile(filename string, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	extension := strings.ToLower(filepath.Ext(filename))
-	if parser, ok := extParsers[extension]; ok {
-		return render(parser, rawBytes, urlPrefix, metas, isWiki)
-	}
-	return nil
-}
-
-// Type returns if markup format via the filename
-func Type(filename string) string {
-	if parser := GetParserByFileName(filename); parser != nil {
-		return parser.Name()
-	}
-	return ""
-}
-
-// IsMarkupFile reports whether file is a markup type file
-func IsMarkupFile(name, markup string) bool {
-	if parser := GetParserByFileName(name); parser != nil {
-		return parser.Name() == markup
-	}
-	return false
-}
-
-// IsReadmeFile reports whether name looks like a README file
-// based on its name. If an extension is provided, it will strictly
-// match that extension.
-// Note that the '.' should be provided in ext, e.g ".md"
-func IsReadmeFile(name string, ext ...string) bool {
-	name = strings.ToLower(name)
-	if len(ext) > 0 {
-		return name == "readme"+ext[0]
-	}
-	if len(name) < 6 {
-		return false
-	} else if len(name) == 6 {
-		return name == "readme"
-	}
-	return name[:7] == "readme."
-}

modules/markup/orgmode/orgmode.go

@@ -8,9 +8,9 @@ import (
 	"bytes"
 	"fmt"
 	"html"
+	"io"
 	"strings"
 
-	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/util"
 
@@ -18,58 +18,62 @@ import (
 )
 
 func init() {
-	markup.RegisterParser(Parser{})
+	markup.RegisterRenderer(Renderer{})
 }
 
-// Parser implements markup.Parser for orgmode
-type Parser struct {
+// Renderer implements markup.Renderer for orgmode
+type Renderer struct {
 }
 
-// Name implements markup.Parser
-func (Parser) Name() string {
+// Name implements markup.Renderer
+func (Renderer) Name() string {
 	return "orgmode"
 }
 
-// NeedPostProcess implements markup.Parser
-func (Parser) NeedPostProcess() bool { return true }
+// NeedPostProcess implements markup.Renderer
+func (Renderer) NeedPostProcess() bool { return true }
 
-// Extensions implements markup.Parser
-func (Parser) Extensions() []string {
+// Extensions implements markup.Renderer
+func (Renderer) Extensions() []string {
 	return []string{".org"}
 }
 
 // Render renders orgmode rawbytes to HTML
-func Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
+func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
 	htmlWriter := org.NewHTMLWriter()
 
-	renderer := &Renderer{
+	w := &Writer{
 		HTMLWriter: htmlWriter,
-		URLPrefix:  urlPrefix,
-		IsWiki:     isWiki,
+		URLPrefix:  ctx.URLPrefix,
+		IsWiki:     ctx.IsWiki,
 	}
 
-	htmlWriter.ExtendingWriter = renderer
+	htmlWriter.ExtendingWriter = w
 
-	res, err := org.New().Silent().Parse(bytes.NewReader(rawBytes), "").Write(renderer)
+	res, err := org.New().Silent().Parse(input, "").Write(w)
 	if err != nil {
-		log.Error("Panic in orgmode.Render: %v Just returning the rawBytes", err)
-		return rawBytes
+		return fmt.Errorf("orgmode.Render failed: %v", err)
 	}
-	return []byte(res)
+	_, err = io.Copy(output, strings.NewReader(res))
+	return err
 }
 
-// RenderString reners orgmode string to HTML string
-func RenderString(rawContent string, urlPrefix string, metas map[string]string, isWiki bool) string {
-	return string(Render([]byte(rawContent), urlPrefix, metas, isWiki))
+// RenderString renders orgmode string to HTML string
+func RenderString(ctx *markup.RenderContext, content string) (string, error) {
+	var buf strings.Builder
+	if err := Render(ctx, strings.NewReader(content), &buf); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
 }
 
-// Render reners orgmode string to HTML string
-func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	return Render(rawBytes, urlPrefix, metas, isWiki)
+// Render renders orgmode string to HTML string
+func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
+	return Render(ctx, input, output)
 }
 
-// Renderer implements org.Writer
-type Renderer struct {
+// Writer implements org.Writer
+type Writer struct {
 	*org.HTMLWriter
 	URLPrefix string
 	IsWiki    bool
@@ -78,7 +82,7 @@ type Renderer struct {
 var byteMailto = []byte("mailto:")
 
 // WriteRegularLink renders images, links or videos
-func (r *Renderer) WriteRegularLink(l org.RegularLink) {
+func (r *Writer) WriteRegularLink(l org.RegularLink) {
 	link := []byte(html.EscapeString(l.URL))
 	if l.Protocol == "file" {
 		link = link[len("file:"):]

modules/markup/orgmode/orgmode_test.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"testing"
 
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 
@@ -23,7 +24,10 @@ func TestRender_StandardLinks(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString(input, setting.AppSubURL, nil, false)
+		buffer, err := RenderString(&markup.RenderContext{
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
@@ -40,7 +44,10 @@ func TestRender_Images(t *testing.T) {
 	setting.AppSubURL = AppSubURL
 
 	test := func(input, expected string) {
-		buffer := RenderString(input, setting.AppSubURL, nil, false)
+		buffer, err := RenderString(&markup.RenderContext{
+			URLPrefix: setting.AppSubURL,
+		}, input)
+		assert.NoError(t, err)
 		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 

modules/markup/renderer.go

@@ -0,0 +1,201 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package markup
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// Init initialize regexps for markdown parsing
+func Init() {
+	getIssueFullPattern()
+	NewSanitizer()
+	if len(setting.Markdown.CustomURLSchemes) > 0 {
+		CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes)
+	}
+
+	// since setting maybe changed extensions, this will reload all renderer extensions mapping
+	extRenderers = make(map[string]Renderer)
+	for _, renderer := range renderers {
+		for _, ext := range renderer.Extensions() {
+			extRenderers[strings.ToLower(ext)] = renderer
+		}
+	}
+}
+
+// RenderContext represents a render context
+type RenderContext struct {
+	Ctx         context.Context
+	Filename    string
+	Type        string
+	IsWiki      bool
+	URLPrefix   string
+	Metas       map[string]string
+	DefaultLink string
+}
+
+// Renderer defines an interface for rendering markup file to HTML
+type Renderer interface {
+	Name() string // markup format name
+	Extensions() []string
+	Render(ctx *RenderContext, input io.Reader, output io.Writer) error
+}
+
+var (
+	extRenderers = make(map[string]Renderer)
+	renderers    = make(map[string]Renderer)
+)
+
+// RegisterRenderer registers a new markup file renderer
+func RegisterRenderer(renderer Renderer) {
+	renderers[renderer.Name()] = renderer
+	for _, ext := range renderer.Extensions() {
+		extRenderers[strings.ToLower(ext)] = renderer
+	}
+}
+
+// GetRendererByFileName get renderer by filename
+func GetRendererByFileName(filename string) Renderer {
+	extension := strings.ToLower(filepath.Ext(filename))
+	return extRenderers[extension]
+}
+
+// GetRendererByType returns a renderer according type
+func GetRendererByType(tp string) Renderer {
+	return renderers[tp]
+}
+
+// Render renders markup file to HTML with all specific handling stuff.
+func Render(ctx *RenderContext, input io.Reader, output io.Writer) error {
+	if ctx.Type != "" {
+		return renderByType(ctx, input, output)
+	} else if ctx.Filename != "" {
+		return renderFile(ctx, input, output)
+	}
+	return errors.New("Render options both filename and type missing")
+}
+
+// RenderString renders Markup string to HTML with all specific handling stuff and return string
+func RenderString(ctx *RenderContext, content string) (string, error) {
+	var buf strings.Builder
+	if err := Render(ctx, strings.NewReader(content), &buf); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+func render(ctx *RenderContext, parser Renderer, input io.Reader, output io.Writer) error {
+	var wg sync.WaitGroup
+	var err error
+	pr, pw := io.Pipe()
+	defer func() {
+		_ = pr.Close()
+		_ = pw.Close()
+	}()
+
+	pr2, pw2 := io.Pipe()
+	defer func() {
+		_ = pr2.Close()
+		_ = pw2.Close()
+	}()
+
+	wg.Add(1)
+	go func() {
+		buf := SanitizeReader(pr2)
+		_, err = io.Copy(output, buf)
+		_ = pr2.Close()
+		wg.Done()
+	}()
+
+	wg.Add(1)
+	go func() {
+		err = PostProcess(ctx, pr, pw2)
+		_ = pr.Close()
+		_ = pw2.Close()
+		wg.Done()
+	}()
+
+	if err1 := parser.Render(ctx, input, pw); err1 != nil {
+		return err1
+	}
+	_ = pw.Close()
+
+	wg.Wait()
+	return err
+}
+
+// ErrUnsupportedRenderType represents
+type ErrUnsupportedRenderType struct {
+	Type string
+}
+
+func (err ErrUnsupportedRenderType) Error() string {
+	return fmt.Sprintf("Unsupported render type: %s", err.Type)
+}
+
+func renderByType(ctx *RenderContext, input io.Reader, output io.Writer) error {
+	if renderer, ok := renderers[ctx.Type]; ok {
+		return render(ctx, renderer, input, output)
+	}
+	return ErrUnsupportedRenderType{ctx.Type}
+}
+
+// ErrUnsupportedRenderExtension represents the error when extension doesn't supported to render
+type ErrUnsupportedRenderExtension struct {
+	Extension string
+}
+
+func (err ErrUnsupportedRenderExtension) Error() string {
+	return fmt.Sprintf("Unsupported render extension: %s", err.Extension)
+}
+
+func renderFile(ctx *RenderContext, input io.Reader, output io.Writer) error {
+	extension := strings.ToLower(filepath.Ext(ctx.Filename))
+	if renderer, ok := extRenderers[extension]; ok {
+		return render(ctx, renderer, input, output)
+	}
+	return ErrUnsupportedRenderExtension{extension}
+}
+
+// Type returns if markup format via the filename
+func Type(filename string) string {
+	if parser := GetRendererByFileName(filename); parser != nil {
+		return parser.Name()
+	}
+	return ""
+}
+
+// IsMarkupFile reports whether file is a markup type file
+func IsMarkupFile(name, markup string) bool {
+	if parser := GetRendererByFileName(name); parser != nil {
+		return parser.Name() == markup
+	}
+	return false
+}
+
+// IsReadmeFile reports whether name looks like a README file
+// based on its name. If an extension is provided, it will strictly
+// match that extension.
+// Note that the '.' should be provided in ext, e.g ".md"
+func IsReadmeFile(name string, ext ...string) bool {
+	name = strings.ToLower(name)
+	if len(ext) > 0 {
+		return name == "readme"+ext[0]
+	}
+	if len(name) < 6 {
+		return false
+	} else if len(name) == 6 {
+		return name == "readme"
+	}
+	return name[:7] == "readme."
+}

modules/notification/mail/mail.go

@@ -104,14 +104,18 @@ func (m *mailNotifier) NotifyIssueChangeAssignee(doer *models.User, issue *model
 	// mail only sent to added assignees and not self-assignee
 	if !removed && doer.ID != assignee.ID && assignee.EmailNotifications() == models.EmailNotificationsEnabled {
 		ct := fmt.Sprintf("Assigned #%d.", issue.Index)
-		mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{assignee})
+		if err := mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{assignee}); err != nil {
+			log.Error("Error in SendIssueAssignedMail for issue[%d] to assignee[%d]: %v", issue.ID, assignee.ID, err)
+		}
 	}
 }
 
 func (m *mailNotifier) NotifyPullReviewRequest(doer *models.User, issue *models.Issue, reviewer *models.User, isRequest bool, comment *models.Comment) {
 	if isRequest && doer.ID != reviewer.ID && reviewer.EmailNotifications() == models.EmailNotificationsEnabled {
 		ct := fmt.Sprintf("Requested to review %s.", issue.HTMLURL())
-		mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{reviewer})
+		if err := mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{reviewer}); err != nil {
+			log.Error("Error in SendIssueAssignedMail for issue[%d] to reviewer[%d]: %v", issue.ID, reviewer.ID, err)
+		}
 	}
 }
 

modules/setting/markup.go

@@ -13,14 +13,14 @@ import (
 	"gopkg.in/ini.v1"
 )
 
-// ExternalMarkupParsers represents the external markup parsers
+// ExternalMarkupRenderers represents the external markup renderers
 var (
-	ExternalMarkupParsers  []MarkupParser
+	ExternalMarkupRenderers []MarkupRenderer
 	ExternalSanitizerRules  []MarkupSanitizerRule
 )
 
-// MarkupParser defines the external parser configured in ini
-type MarkupParser struct {
+// MarkupRenderer defines the external parser configured in ini
+type MarkupRenderer struct {
 	Enabled         bool
 	MarkupName      string
 	Command         string
@@ -124,7 +124,7 @@ func newMarkupRenderer(name string, sec *ini.Section) {
 		return
 	}
 
-	ExternalMarkupParsers = append(ExternalMarkupParsers, MarkupParser{
+	ExternalMarkupRenderers = append(ExternalMarkupRenderers, MarkupRenderer{
 		Enabled:         sec.Key("ENABLED").MustBool(false),
 		MarkupName:      name,
 		FileExtensions:  exts,

modules/templates/helper.go

@@ -665,7 +665,11 @@ func RenderCommitMessageLink(msg, urlPrefix, urlDefault string, metas map[string
 	cleanMsg := template.HTMLEscapeString(msg)
 	// we can safely assume that it will not return any error, since there
 	// shouldn't be any special HTML.
-	fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, urlDefault, metas)
+	fullMessage, err := markup.RenderCommitMessage(&markup.RenderContext{
+		URLPrefix:   urlPrefix,
+		DefaultLink: urlDefault,
+		Metas:       metas,
+	}, cleanMsg)
 	if err != nil {
 		log.Error("RenderCommitMessage: %v", err)
 		return ""
@@ -692,7 +696,11 @@ func RenderCommitMessageLinkSubject(msg, urlPrefix, urlDefault string, metas map
 
 	// we can safely assume that it will not return any error, since there
 	// shouldn't be any special HTML.
-	renderedMessage, err := markup.RenderCommitMessageSubject([]byte(template.HTMLEscapeString(msgLine)), urlPrefix, urlDefault, metas)
+	renderedMessage, err := markup.RenderCommitMessageSubject(&markup.RenderContext{
+		URLPrefix:   urlPrefix,
+		DefaultLink: urlDefault,
+		Metas:       metas,
+	}, template.HTMLEscapeString(msgLine))
 	if err != nil {
 		log.Error("RenderCommitMessageSubject: %v", err)
 		return template.HTML("")
@@ -714,7 +722,10 @@ func RenderCommitBody(msg, urlPrefix string, metas map[string]string) template.H
 		return template.HTML("")
 	}
 
-	renderedMessage, err := markup.RenderCommitMessage([]byte(template.HTMLEscapeString(msgLine)), urlPrefix, "", metas)
+	renderedMessage, err := markup.RenderCommitMessage(&markup.RenderContext{
+		URLPrefix: urlPrefix,
+		Metas:     metas,
+	}, template.HTMLEscapeString(msgLine))
 	if err != nil {
 		log.Error("RenderCommitMessage: %v", err)
 		return ""
@@ -724,7 +735,10 @@ func RenderCommitBody(msg, urlPrefix string, metas map[string]string) template.H
 
 // RenderIssueTitle renders issue/pull title with defined post processors
 func RenderIssueTitle(text, urlPrefix string, metas map[string]string) template.HTML {
-	renderedText, err := markup.RenderIssueTitle([]byte(template.HTMLEscapeString(text)), urlPrefix, metas)
+	renderedText, err := markup.RenderIssueTitle(&markup.RenderContext{
+		URLPrefix: urlPrefix,
+		Metas:     metas,
+	}, template.HTMLEscapeString(text))
 	if err != nil {
 		log.Error("RenderIssueTitle: %v", err)
 		return template.HTML("")
@@ -734,7 +748,7 @@ func RenderIssueTitle(text, urlPrefix string, metas map[string]string) template.
 
 // RenderEmoji renders html text with emoji post processors
 func RenderEmoji(text string) template.HTML {
-	renderedText, err := markup.RenderEmoji([]byte(template.HTMLEscapeString(text)))
+	renderedText, err := markup.RenderEmoji(template.HTMLEscapeString(text))
 	if err != nil {
 		log.Error("RenderEmoji: %v", err)
 		return template.HTML("")
@@ -758,7 +772,10 @@ func ReactionToEmoji(reaction string) template.HTML {
 // RenderNote renders the contents of a git-notes file as a commit message.
 func RenderNote(msg, urlPrefix string, metas map[string]string) template.HTML {
 	cleanMsg := template.HTMLEscapeString(msg)
-	fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, "", metas)
+	fullMessage, err := markup.RenderCommitMessage(&markup.RenderContext{
+		URLPrefix: urlPrefix,
+		Metas:     metas,
+	}, cleanMsg)
 	if err != nil {
 		log.Error("RenderNote: %v", err)
 		return ""

routers/api/v1/misc/markdown.go

@@ -5,11 +5,11 @@
 package misc
 
 import (
-	"io/ioutil"
 	"net/http"
 	"strings"
 
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 	api "code.gitea.io/gitea/modules/structs"
@@ -55,7 +55,6 @@ func Markdown(ctx *context.APIContext) {
 	case "comment":
 		fallthrough
 	case "gfm":
-		md := []byte(form.Text)
 		urlPrefix := form.Context
 		meta := map[string]string{}
 		if !strings.HasPrefix(setting.AppSubURL+"/", urlPrefix) {
@@ -77,22 +76,19 @@ func Markdown(ctx *context.APIContext) {
 		if form.Mode == "gfm" {
 			meta["mode"] = "document"
 		}
-		if form.Wiki {
-			_, err := ctx.Write([]byte(markdown.RenderWiki(md, urlPrefix, meta)))
-			if err != nil {
+
+		if err := markdown.Render(&markup.RenderContext{
+			URLPrefix: urlPrefix,
+			Metas:     meta,
+			IsWiki:    form.Wiki,
+		}, strings.NewReader(form.Text), ctx.Resp); err != nil {
 			ctx.InternalServerError(err)
 			return
 		}
-		} else {
-			_, err := ctx.Write(markdown.Render(md, urlPrefix, meta))
-			if err != nil {
-				ctx.InternalServerError(err)
-				return
-			}
-		}
 	default:
-		_, err := ctx.Write(markdown.RenderRaw([]byte(form.Text), "", false))
-		if err != nil {
+		if err := markdown.RenderRaw(&markup.RenderContext{
+			URLPrefix: form.Context,
+		}, strings.NewReader(form.Text), ctx.Resp); err != nil {
 			ctx.InternalServerError(err)
 			return
 		}
@@ -120,14 +116,8 @@ func MarkdownRaw(ctx *context.APIContext) {
 	//     "$ref": "#/responses/MarkdownRender"
 	//   "422":
 	//     "$ref": "#/responses/validationError"
-
-	body, err := ioutil.ReadAll(ctx.Req.Body)
-	if err != nil {
-		ctx.Error(http.StatusUnprocessableEntity, "", err)
-		return
-	}
-	_, err = ctx.Write(markdown.RenderRaw(body, "", false))
-	if err != nil {
+	defer ctx.Req.Body.Close()
+	if err := markdown.RenderRaw(&markup.RenderContext{}, ctx.Req.Body, ctx.Resp); err != nil {
 		ctx.InternalServerError(err)
 		return
 	}

routers/init.go

@@ -143,7 +143,7 @@ func GlobalInit(ctx context.Context) {
 	NewServices()
 
 	highlight.NewContext()
-	external.RegisterParsers()
+	external.RegisterRenderers()
 	markup.Init()
 
 	if setting.EnableSQLite3 {

routers/org/home.go

@@ -11,6 +11,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 )
@@ -37,7 +38,15 @@ func Home(ctx *context.Context) {
 	ctx.Data["PageIsUserProfile"] = true
 	ctx.Data["Title"] = org.DisplayName()
 	if len(org.Description) != 0 {
-		ctx.Data["RenderedDescription"] = string(markdown.Render([]byte(org.Description), ctx.Repo.RepoLink, map[string]string{"mode": "document"}))
+		desc, err := markdown.RenderString(&markup.RenderContext{
+			URLPrefix: ctx.Repo.RepoLink,
+			Metas:     map[string]string{"mode": "document"},
+		}, org.Description)
+		if err != nil {
+			ctx.ServerError("RenderString", err)
+			return
+		}
+		ctx.Data["RenderedDescription"] = desc
 	}
 
 	var orderBy models.SearchOrderBy

routers/repo/compare.go

@@ -10,7 +10,6 @@ import (
 	"errors"
 	"fmt"
 	"html"
-	"io/ioutil"
 	"net/http"
 	"path"
 	"path/filepath"
@@ -117,14 +116,7 @@ func setCsvCompareContext(ctx *context.Context) {
 			}
 			defer reader.Close()
 
-			b, err := ioutil.ReadAll(reader)
-			if err != nil {
-				return nil, err
-			}
-
-			b = charset.ToUTF8WithFallback(b)
-
-			return csv_module.CreateReaderAndGuessDelimiter(b), nil
+			return csv_module.CreateReaderAndGuessDelimiter(charset.ToUTF8WithFallbackReader(reader))
 		}
 
 		baseReader, err := csvReaderFromCommit(baseCommit)

routers/repo/issue.go

@@ -1131,8 +1131,14 @@ func ViewIssue(ctx *context.Context) {
 	}
 	ctx.Data["IssueWatch"] = iw
 
-	issue.RenderedContent = string(markdown.Render([]byte(issue.Content), ctx.Repo.RepoLink,
-		ctx.Repo.Repository.ComposeMetas()))
+	issue.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+		URLPrefix: ctx.Repo.RepoLink,
+		Metas:     ctx.Repo.Repository.ComposeMetas(),
+	}, issue.Content)
+	if err != nil {
+		ctx.ServerError("RenderString", err)
+		return
+	}
 
 	repo := ctx.Repo.Repository
 
@@ -1289,9 +1295,14 @@ func ViewIssue(ctx *context.Context) {
 				return
 			}
 
-			comment.RenderedContent = string(markdown.Render([]byte(comment.Content), ctx.Repo.RepoLink,
-				ctx.Repo.Repository.ComposeMetas()))
-
+			comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+				URLPrefix: ctx.Repo.RepoLink,
+				Metas:     ctx.Repo.Repository.ComposeMetas(),
+			}, comment.Content)
+			if err != nil {
+				ctx.ServerError("RenderString", err)
+				return
+			}
 			// Check tag.
 			tag, ok = marked[comment.PosterID]
 			if ok {
@@ -1359,8 +1370,14 @@ func ViewIssue(ctx *context.Context) {
 				}
 			}
 		} else if comment.Type == models.CommentTypeCode || comment.Type == models.CommentTypeReview || comment.Type == models.CommentTypeDismissReview {
-			comment.RenderedContent = string(markdown.Render([]byte(comment.Content), ctx.Repo.RepoLink,
-				ctx.Repo.Repository.ComposeMetas()))
+			comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+				URLPrefix: ctx.Repo.RepoLink,
+				Metas:     ctx.Repo.Repository.ComposeMetas(),
+			}, comment.Content)
+			if err != nil {
+				ctx.ServerError("RenderString", err)
+				return
+			}
 			if err = comment.LoadReview(); err != nil && !models.IsErrReviewNotExist(err) {
 				ctx.ServerError("LoadReview", err)
 				return
@@ -1708,10 +1725,20 @@ func UpdateIssueContent(ctx *context.Context) {
 	files := ctx.QueryStrings("files[]")
 	if err := updateAttachments(issue, files); err != nil {
 		ctx.ServerError("UpdateAttachments", err)
+		return
+	}
+
+	content, err := markdown.RenderString(&markup.RenderContext{
+		URLPrefix: ctx.Query("context"),
+		Metas:     ctx.Repo.Repository.ComposeMetas(),
+	}, issue.Content)
+	if err != nil {
+		ctx.ServerError("RenderString", err)
+		return
 	}
 
 	ctx.JSON(http.StatusOK, map[string]interface{}{
-		"content":     string(markdown.Render([]byte(issue.Content), ctx.Query("context"), ctx.Repo.Repository.ComposeMetas())),
+		"content":     content,
 		"attachments": attachmentsHTML(ctx, issue.Attachments, issue.Content),
 	})
 }
@@ -2125,10 +2152,20 @@ func UpdateCommentContent(ctx *context.Context) {
 	files := ctx.QueryStrings("files[]")
 	if err := updateAttachments(comment, files); err != nil {
 		ctx.ServerError("UpdateAttachments", err)
+		return
+	}
+
+	content, err := markdown.RenderString(&markup.RenderContext{
+		URLPrefix: ctx.Query("context"),
+		Metas:     ctx.Repo.Repository.ComposeMetas(),
+	}, comment.Content)
+	if err != nil {
+		ctx.ServerError("RenderString", err)
+		return
 	}
 
 	ctx.JSON(http.StatusOK, map[string]interface{}{
-		"content":     string(markdown.Render([]byte(comment.Content), ctx.Query("context"), ctx.Repo.Repository.ComposeMetas())),
+		"content":     content,
 		"attachments": attachmentsHTML(ctx, comment.Attachments, comment.Content),
 	})
 }

routers/repo/lfs.go

@@ -296,20 +296,13 @@ func LFSFileGet(ctx *context.Context) {
 			break
 		}
 
-		d, _ := ioutil.ReadAll(dataRc)
-		buf = charset.ToUTF8WithFallback(append(buf, d...))
+		buf := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
 
 		// Building code view blocks with line number on server side.
-		var fileContent string
-		if content, err := charset.ToUTF8WithErr(buf); err != nil {
-			log.Error("ToUTF8WithErr: %v", err)
-			fileContent = string(buf)
-		} else {
-			fileContent = content
-		}
+		fileContent, _ := ioutil.ReadAll(buf)
 
 		var output bytes.Buffer
-		lines := strings.Split(fileContent, "\n")
+		lines := strings.Split(string(fileContent), "\n")
 		//Remove blank line at the end of file
 		if len(lines) > 0 && lines[len(lines)-1] == "" {
 			lines = lines[:len(lines)-1]

routers/repo/milestone.go

@@ -12,6 +12,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
@@ -84,7 +85,14 @@ func Milestones(ctx *context.Context) {
 		}
 	}
 	for _, m := range miles {
-		m.RenderedContent = string(markdown.Render([]byte(m.Content), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()))
+		m.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+			URLPrefix: ctx.Repo.RepoLink,
+			Metas:     ctx.Repo.Repository.ComposeMetas(),
+		}, m.Content)
+		if err != nil {
+			ctx.ServerError("RenderString", err)
+			return
+		}
 	}
 	ctx.Data["Milestones"] = miles
 
@@ -269,7 +277,14 @@ func MilestoneIssuesAndPulls(ctx *context.Context) {
 		return
 	}
 
-	milestone.RenderedContent = string(markdown.Render([]byte(milestone.Content), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()))
+	milestone.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+		URLPrefix: ctx.Repo.RepoLink,
+		Metas:     ctx.Repo.Repository.ComposeMetas(),
+	}, milestone.Content)
+	if err != nil {
+		ctx.ServerError("RenderString", err)
+		return
+	}
 
 	ctx.Data["Title"] = milestone.Name
 	ctx.Data["Milestone"] = milestone