Restructure markup & markdown to prepare for multiple markup language… (#2411)

* restructure markup & markdown to prepare for multiple markup languages support

* adjust some functions between markdown and markup

* fix tests

* improve the comments

models/issue_comment.go

@@ -16,7 +16,7 @@ import (
 	api "code.gitea.io/sdk/gitea"
 
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/markup"
 )
 
 // CommentType defines whether a comment is just a simple comment, an action (like close) or a reference.
@@ -272,7 +272,7 @@ func (c *Comment) LoadAssignees() error {
 // MailParticipants sends new comment emails to repository watchers
 // and mentioned people.
 func (c *Comment) MailParticipants(e Engine, opType ActionType, issue *Issue) (err error) {
-	mentions := markdown.FindAllMentions(c.Content)
+	mentions := markup.FindAllMentions(c.Content)
 	if err = UpdateIssueMentions(e, c.IssueID, mentions); err != nil {
 		return fmt.Errorf("UpdateIssueMentions [%d]: %v", c.IssueID, err)
 	}

models/issue_mail.go

@@ -10,7 +10,7 @@ import (
 	"github.com/Unknwon/com"
 
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 )
 
@@ -104,7 +104,7 @@ func (issue *Issue) MailParticipants() (err error) {
 }
 
 func (issue *Issue) mailParticipants(e Engine) (err error) {
-	mentions := markdown.FindAllMentions(issue.Content)
+	mentions := markup.FindAllMentions(issue.Content)
 	if err = UpdateIssueMentions(e, issue.ID, mentions); err != nil {
 		return fmt.Errorf("UpdateIssueMentions [%d]: %v", issue.ID, err)
 	}

models/mail.go

@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/mailer"
 	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 	"gopkg.in/gomail.v2"
 	"gopkg.in/macaron.v1"
@@ -150,7 +151,7 @@ func composeTplData(subject, body, link string) map[string]interface{} {
 
 func composeIssueCommentMessage(issue *Issue, doer *User, comment *Comment, tplName base.TplName, tos []string, info string) *mailer.Message {
 	subject := issue.mailSubject()
-	body := string(markdown.RenderString(issue.Content, issue.Repo.HTMLURL(), issue.Repo.ComposeMetas()))
+	body := string(markup.RenderByType(markdown.MarkupName, []byte(issue.Content), issue.Repo.HTMLURL(), issue.Repo.ComposeMetas()))
 
 	data := make(map[string]interface{}, 10)
 	if comment != nil {

models/migrations/v16.go

@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"time"
 
-	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/markup"
 
 	"github.com/go-xorm/xorm"
 )
@@ -101,7 +101,7 @@ func addUnitsToTables(x *xorm.Engine) error {
 				config["ExternalTrackerURL"] = repo.ExternalTrackerURL
 				config["ExternalTrackerFormat"] = repo.ExternalTrackerFormat
 				if len(repo.ExternalTrackerStyle) == 0 {
-					repo.ExternalTrackerStyle = markdown.IssueNameStyleNumeric
+					repo.ExternalTrackerStyle = markup.IssueNameStyleNumeric
 				}
 				config["ExternalTrackerStyle"] = repo.ExternalTrackerStyle
 			case V16UnitTypeExternalWiki:

models/repo.go

@@ -22,7 +22,7 @@ import (
 
 	"code.gitea.io/git"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/setting"
@@ -480,10 +480,10 @@ func (repo *Repository) ComposeMetas() map[string]string {
 			"repo":   repo.Name,
 		}
 		switch unit.ExternalTrackerConfig().ExternalTrackerStyle {
-		case markdown.IssueNameStyleAlphanumeric:
-			repo.ExternalMetas["style"] = markdown.IssueNameStyleAlphanumeric
+		case markup.IssueNameStyleAlphanumeric:
+			repo.ExternalMetas["style"] = markup.IssueNameStyleAlphanumeric
 		default:
-			repo.ExternalMetas["style"] = markdown.IssueNameStyleNumeric
+			repo.ExternalMetas["style"] = markup.IssueNameStyleNumeric
 		}
 
 	}
@@ -708,7 +708,7 @@ func (repo *Repository) DescriptionHTML() template.HTML {
 	sanitize := func(s string) string {
 		return fmt.Sprintf(`<a href="%[1]s" target="_blank" rel="noopener">%[1]s</a>`, s)
 	}
-	return template.HTML(descPattern.ReplaceAllStringFunc(markdown.Sanitize(repo.Description), sanitize))
+	return template.HTML(descPattern.ReplaceAllStringFunc(markup.Sanitize(repo.Description), sanitize))
 }
 
 // LocalCopyPath returns the local repository copy path

models/repo_test.go

@@ -8,7 +8,7 @@ import (
 	"path"
 	"testing"
 
-	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/Unknwon/com"
@@ -39,13 +39,13 @@ func TestRepo(t *testing.T) {
 		assert.Equal(t, "https://someurl.com/{user}/{repo}/{issue}", metas["format"])
 	}
 
-	testSuccess(markdown.IssueNameStyleNumeric)
+	testSuccess(markup.IssueNameStyleNumeric)
 
-	externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markdown.IssueNameStyleAlphanumeric
-	testSuccess(markdown.IssueNameStyleAlphanumeric)
+	externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markup.IssueNameStyleAlphanumeric
+	testSuccess(markup.IssueNameStyleAlphanumeric)
 
-	externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markdown.IssueNameStyleNumeric
-	testSuccess(markdown.IssueNameStyleNumeric)
+	externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markup.IssueNameStyleNumeric
+	testSuccess(markup.IssueNameStyleNumeric)
 }
 
 func TestGetRepositoryCount(t *testing.T) {

modules/markup/markup.go

@@ -9,6 +9,12 @@ import (
 	"strings"
 )
 
+// Init initialize regexps for markdown parsing
+func Init() {
+	getIssueFullPattern()
+	NewSanitizer()
+}
+
 // Parser defines an interface for parsering markup file to HTML
 type Parser interface {
 	Name() string // markup format name
@@ -17,66 +23,94 @@ type Parser interface {
 }
 
 var (
-	parsers = make(map[string]Parser)
+	extParsers = make(map[string]Parser)
+	parsers    = make(map[string]Parser)
 )
 
 // RegisterParser registers a new markup file parser
 func RegisterParser(parser Parser) {
+	parsers[parser.Name()] = parser
 	for _, ext := range parser.Extensions() {
-		parsers[strings.ToLower(ext)] = parser
+		extParsers[strings.ToLower(ext)] = parser
 	}
 }
 
+// GetParserByFileName get parser by filename
+func GetParserByFileName(filename string) Parser {
+	extension := strings.ToLower(filepath.Ext(filename))
+	return extParsers[extension]
+}
+
+// GetParserByType returns a parser according type
+func GetParserByType(tp string) Parser {
+	return parsers[tp]
+}
+
 // Render renders markup file to HTML with all specific handling stuff.
 func Render(filename string, rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
-	return render(filename, rawBytes, urlPrefix, metas, false)
+	return renderFile(filename, rawBytes, urlPrefix, metas, false)
 }
 
-func render(filename string, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
-	extension := strings.ToLower(filepath.Ext(filename))
-	if parser, ok := parsers[extension]; ok {
-		return parser.Render(rawBytes, urlPrefix, metas, isWiki)
-	}
-	return nil
+// RenderByType renders markup to HTML with special links and returns string type.
+func RenderByType(tp string, rawBytes []byte, urlPrefix string, metas map[string]string) []byte {
+	return renderByType(tp, rawBytes, urlPrefix, metas, false)
 }
 
 // RenderString renders Markdown to HTML with special links and returns string type.
 func RenderString(filename string, raw, urlPrefix string, metas map[string]string) string {
-	return string(render(filename, []byte(raw), urlPrefix, metas, false))
+	return string(renderFile(filename, []byte(raw), urlPrefix, metas, false))
 }
 
 // RenderWiki renders markdown wiki page to HTML and return HTML string
 func RenderWiki(filename string, rawBytes []byte, urlPrefix string, metas map[string]string) string {
-	return string(render(filename, rawBytes, urlPrefix, metas, true))
+	return string(renderFile(filename, rawBytes, urlPrefix, metas, true))
+}
+
+func render(parser Parser, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
+	urlPrefix = strings.Replace(urlPrefix, " ", "+", -1)
+	result := parser.Render(rawBytes, urlPrefix, metas, isWiki)
+	result = PostProcess(result, urlPrefix, metas, isWiki)
+	return SanitizeBytes(result)
+}
+
+func renderByType(tp string, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
+	if parser, ok := parsers[tp]; ok {
+		return render(parser, rawBytes, urlPrefix, metas, isWiki)
+	}
+	return nil
+}
+
+func renderFile(filename string, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
+	extension := strings.ToLower(filepath.Ext(filename))
+	if parser, ok := extParsers[extension]; ok {
+		return render(parser, rawBytes, urlPrefix, metas, isWiki)
+	}
+	return nil
 }
 
 // Type returns if markup format via the filename
 func Type(filename string) string {
-	extension := strings.ToLower(filepath.Ext(filename))
-	if parser, ok := parsers[extension]; ok {
+	if parser := GetParserByFileName(filename); parser != nil {
 		return parser.Name()
 	}
 	return ""
 }
 
-// ReadmeFileType reports whether name looks like a README file
-// based on its name and find the parser via its ext name
-func ReadmeFileType(name string) (string, bool) {
-	if IsReadmeFile(name) {
-		return Type(name), true
+// IsMarkupFile reports whether file is a markup type file
+func IsMarkupFile(name, markup string) bool {
+	if parser := GetParserByFileName(name); parser != nil {
+		return parser.Name() == markup
 	}
-	return "", false
+	return false
 }
 
 // IsReadmeFile reports whether name looks like a README file
 // based on its name.
 func IsReadmeFile(name string) bool {
+	name = strings.ToLower(name)
 	if len(name) < 6 {
 		return false
-	}
-
-	name = strings.ToLower(name)
-	if len(name) == 6 {
+	} else if len(name) == 6 {
 		return name == "readme"
 	}
 	return name[:7] == "readme."

modules/markup/markup_test.go

@@ -2,11 +2,14 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
-package markup
+package markup_test
 
 import (
 	"testing"
 
+	_ "code.gitea.io/gitea/modules/markdown"
+	. "code.gitea.io/gitea/modules/markup"
+
 	"github.com/stretchr/testify/assert"
 )
 

modules/markup/sanitizer.go

@@ -3,7 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
-package markdown
+package markup
 
 import (
 	"regexp"

modules/markup/sanitizer_test.go

@@ -3,7 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
-package markdown
+package markup
 
 import (
 	"testing"

modules/templates/helper.go

@@ -24,7 +24,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 )
 
@@ -173,7 +173,7 @@ func SafeJS(raw string) template.JS {
 
 // Str2html render Markdown text to HTML
 func Str2html(raw string) template.HTML {
-	return template.HTML(markdown.Sanitize(raw))
+	return template.HTML(markup.Sanitize(raw))
 }
 
 // List traversings the list
@@ -253,7 +253,7 @@ func ReplaceLeft(s, old, new string) string {
 // RenderCommitMessage renders commit message with XSS-safe and special links.
 func RenderCommitMessage(full bool, msg, urlPrefix string, metas map[string]string) template.HTML {
 	cleanMsg := template.HTMLEscapeString(msg)
-	fullMessage := string(markdown.RenderIssueIndexPattern([]byte(cleanMsg), urlPrefix, metas))
+	fullMessage := string(markup.RenderIssueIndexPattern([]byte(cleanMsg), urlPrefix, metas))
 	msgLines := strings.Split(strings.TrimSpace(fullMessage), "\n")
 	numLines := len(msgLines)
 	if numLines == 0 {

routers/api/v1/misc/markdown.go

@@ -9,6 +9,7 @@ import (
 
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 )
 
@@ -39,7 +40,7 @@ func Markdown(ctx *context.APIContext, form api.MarkdownOption) {
 	switch form.Mode {
 	case "gfm":
 		md := []byte(form.Text)
-		context := markdown.URLJoin(setting.AppURL, form.Context)
+		context := markup.URLJoin(setting.AppURL, form.Context)
 		if form.Wiki {
 			ctx.Write([]byte(markdown.RenderWiki(md, context, nil)))
 		} else {

routers/api/v1/misc/markdown_test.go

@@ -1,23 +1,21 @@
 package misc
 
 import (
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
+	"strings"
 	"testing"
 
-	macaron "gopkg.in/macaron.v1"
-
-	"net/url"
-
-	"io/ioutil"
-	"strings"
-
 	"code.gitea.io/gitea/modules/context"
-	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 	api "code.gitea.io/sdk/gitea"
+
 	"github.com/go-macaron/inject"
 	"github.com/stretchr/testify/assert"
+	macaron "gopkg.in/macaron.v1"
 )
 
 const AppURL = "http://localhost:3000/"
@@ -55,7 +53,7 @@ func TestAPI_RenderGFM(t *testing.T) {
 		Context: Repo,
 		Wiki:    true,
 	}
-	requrl, _ := url.Parse(markdown.URLJoin(AppURL, "api", "v1", "markdown"))
+	requrl, _ := url.Parse(markup.URLJoin(AppURL, "api", "v1", "markdown"))
 	req := &http.Request{
 		Method: "POST",
 		URL:    requrl,
@@ -149,7 +147,7 @@ func TestAPI_RenderSimple(t *testing.T) {
 		Text:    "",
 		Context: Repo,
 	}
-	requrl, _ := url.Parse(markdown.URLJoin(AppURL, "api", "v1", "markdown"))
+	requrl, _ := url.Parse(markup.URLJoin(AppURL, "api", "v1", "markdown"))
 	req := &http.Request{
 		Method: "POST",
 		URL:    requrl,
@@ -168,7 +166,7 @@ func TestAPI_RenderSimple(t *testing.T) {
 func TestAPI_RenderRaw(t *testing.T) {
 	setting.AppURL = AppURL
 
-	requrl, _ := url.Parse(markdown.URLJoin(AppURL, "api", "v1", "markdown"))
+	requrl, _ := url.Parse(markup.URLJoin(AppURL, "api", "v1", "markdown"))
 	req := &http.Request{
 		Method: "POST",
 		URL:    requrl,

routers/init.go

@@ -16,7 +16,7 @@ import (
 	"code.gitea.io/gitea/modules/indexer"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/mailer"
-	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/ssh"
 	macaron "gopkg.in/macaron.v1"
@@ -50,8 +50,8 @@ func GlobalInit() {
 
 	if setting.InstallLock {
 		highlight.NewContext()
-		markdown.InitMarkdown()
-		markdown.NewSanitizer()
+		markup.Init()
+
 		if err := models.NewEngine(migrations.Migrate); err != nil {
 			log.Fatal(4, "Failed to initialize ORM engine: %v", err)
 		}

routers/repo/view.go

@@ -61,13 +61,12 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 			continue
 		}
 
-		tp, ok := markup.ReadmeFileType(entry.Name())
-		if !ok {
+		if !markup.IsReadmeFile(entry.Name()) {
 			continue
 		}
 
 		readmeFile = entry.Blob()
-		if tp != "" {
+		if markup.Type(entry.Name()) != "" {
 			break
 		}
 	}