lianguan/gitea
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Rework special link parsing in the post-processing of markup (#3354)

Browse Source 
* Get rid of autolink

* autolink in markdown

* Replace email addresses with mailto links

* better handling of links

* Remove autolink.js from footer

* Refactor entire html.go

* fix some bugs

* Make tests green, move what we can to html_internal_test, various other changes to processor logic

* Make markdown tests work again

This is just a description to allow me to force push in order to restart
the drone build.

* Fix failing markdown tests in routers/api/v1/misc

* Add license headers, log errors, future-proof <body>

* fix formatting
This commit is contained in:
Morgan Bazalgette
2018-02-27 08:09:18 +01:00
committed by Lauris BH
parent 769ab1e424
commit 535445c32e
12 changed files with 1029 additions and 1025 deletions
Download Patch File Download Diff File Expand all files Collapse all files

798
modules/markup/html.go
View File

File diff suppressed because it is too large Load Diff

382
modules/markup/html_internal_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

465
modules/markup/html_test.go
View File

File diff suppressed because it is too large Load Diff

40
modules/markup/markdown/markdown.go
View File

@ -22,10 +22,14 @@ type Renderer struct {
IsWiki bool
}
var byteMailto = []byte("mailto:")
// Link defines how formal links should be processed to produce corresponding HTML elements.
func (r *Renderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
if len(link) > 0 && !markup.IsLink(link) {
if link[0] != '#' {
// special case: this is not a link, a hash link or a mailto:, so it's a
// relative URL
if len(link) > 0 && !markup.IsLink(link) &&
link[0] != '#' && !bytes.HasPrefix(link, byteMailto) {
lnk := string(link)
if r.IsWiki {
lnk = util.URLJoin("wiki", lnk)
@ -33,7 +37,6 @@ func (r *Renderer) Link(out *bytes.Buffer, link []byte, title []byte, content []
mLink := util.URLJoin(r.URLPrefix, lnk)
link = []byte(mLink)
}
}
r.Renderer.Link(out, link, title, content)
}
@ -124,30 +127,33 @@ func (r *Renderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byt
out.WriteString("</a>")
}
const (
blackfridayExtensions = 0 |
blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
blackfriday.EXTENSION_TABLES |
blackfriday.EXTENSION_FENCED_CODE |
blackfriday.EXTENSION_STRIKETHROUGH |
blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK
blackfridayHTMLFlags = 0 |
blackfriday.HTML_SKIP_STYLE |
blackfriday.HTML_OMIT_CONTENTS |
blackfriday.HTML_USE_SMARTYPANTS
)
// RenderRaw renders Markdown to HTML without handling special links.
func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte {
htmlFlags := 0
htmlFlags |= blackfriday.HTML_SKIP_STYLE
htmlFlags |= blackfriday.HTML_OMIT_CONTENTS
renderer := &Renderer{
Renderer: blackfriday.HtmlRenderer(htmlFlags, "", ""),
Renderer: blackfriday.HtmlRenderer(blackfridayHTMLFlags, "", ""),
URLPrefix: urlPrefix,
IsWiki: wikiMarkdown,
}
// set up the parser
extensions := 0
extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
extensions |= blackfriday.EXTENSION_TABLES
extensions |= blackfriday.EXTENSION_FENCED_CODE
extensions |= blackfriday.EXTENSION_STRIKETHROUGH
extensions |= blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK
exts := blackfridayExtensions
if setting.Markdown.EnableHardLineBreak {
extensions |= blackfriday.EXTENSION_HARD_LINE_BREAK
exts |= blackfriday.EXTENSION_HARD_LINE_BREAK
}
body = blackfriday.Markdown(body, renderer, extensions)
body = blackfriday.Markdown(body, renderer, exts)
return body
}

102
modules/markup/markdown/markdown_test.go
View File

@ -8,7 +8,6 @@ import (
"strings"
"testing"
"code.gitea.io/gitea/modules/markup"
. "code.gitea.io/gitea/modules/markup/markdown"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@ -41,69 +40,6 @@ func TestRender_StandardLinks(t *testing.T) {
`<p><a href="`+lnkWiki+`" rel="nofollow">WikiPage</a></p>`)
}
func TestRender_ShortLinks(t *testing.T) {
setting.AppURL = AppURL
setting.AppSubURL = AppSubURL
tree := util.URLJoin(AppSubURL, "src", "master")
test := func(input, expected, expectedWiki string) {
buffer := RenderString(input, tree, nil)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
buffer = RenderWiki([]byte(input), setting.AppSubURL, nil)
assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(string(buffer)))
}
rawtree := util.URLJoin(AppSubURL, "raw", "master")
url := util.URLJoin(tree, "Link")
otherUrl := util.URLJoin(tree, "OtherLink")
imgurl := util.URLJoin(rawtree, "Link.jpg")
urlWiki := util.URLJoin(AppSubURL, "wiki", "Link")
otherUrlWiki := util.URLJoin(AppSubURL, "wiki", "OtherLink")
imgurlWiki := util.URLJoin(AppSubURL, "wiki", "raw", "Link.jpg")
favicon := "http://google.com/favicon.ico"
test(
"[[Link]]",
`<p><a href="`+url+`" rel="nofollow">Link</a></p>`,
`<p><a href="`+urlWiki+`" rel="nofollow">Link</a></p>`)
test(
"[[Link.jpg]]",
`<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="Link.jpg" title="Link.jpg"/></a></p>`,
`<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="Link.jpg" title="Link.jpg"/></a></p>`)
test(
"[["+favicon+"]]",
`<p><a href="`+favicon+`" rel="nofollow"><img src="`+favicon+`" title="favicon.ico"/></a></p>`,
`<p><a href="`+favicon+`" rel="nofollow"><img src="`+favicon+`" title="favicon.ico"/></a></p>`)
test(
"[[Name|Link]]",
`<p><a href="`+url+`" rel="nofollow">Name</a></p>`,
`<p><a href="`+urlWiki+`" rel="nofollow">Name</a></p>`)
test(
"[[Name|Link.jpg]]",
`<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="Name" title="Name"/></a></p>`,
`<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="Name" title="Name"/></a></p>`)
test(
"[[Name|Link.jpg|alt=AltName]]",
`<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="AltName" title="AltName"/></a></p>`,
`<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="AltName" title="AltName"/></a></p>`)
test(
"[[Name|Link.jpg|title=Title]]",
`<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="Title" title="Title"/></a></p>`,
`<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="Title" title="Title"/></a></p>`)
test(
"[[Name|Link.jpg|alt=AltName|title=Title]]",
`<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="AltName" title="Title"/></a></p>`,
`<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="AltName" title="Title"/></a></p>`)
test(
"[[Name|Link.jpg|alt=\"AltName\"|title='Title']]",
`<p><a href="`+imgurl+`" rel="nofollow"><img src="`+imgurl+`" alt="AltName" title="Title"/></a></p>`,
`<p><a href="`+imgurlWiki+`" rel="nofollow"><img src="`+imgurlWiki+`" alt="AltName" title="Title"/></a></p>`)
test(
"[[Link]] [[OtherLink]]",
`<p><a href="`+url+`" rel="nofollow">Link</a> <a href="`+otherUrl+`" rel="nofollow">OtherLink</a></p>`,
`<p><a href="`+urlWiki+`" rel="nofollow">Link</a> <a href="`+otherUrlWiki+`" rel="nofollow">OtherLink</a></p>`)
}
func TestMisc_IsMarkdownFile(t *testing.T) {
setting.Markdown.FileExtensions = []string{".md", ".markdown", ".mdown", ".mkd"}
trueTestCases := []string{
@ -141,35 +77,11 @@ func TestRender_Images(t *testing.T) {
test(
"!["+title+"]("+url+")",
`<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"></a></p>`)
`<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" alt="`+title+`"/></a></p>`)
test(
"[["+title+"|"+url+"]]",
`<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" alt="`+title+`" title="`+title+`"/></a></p>`)
}
func TestRegExp_ShortLinkPattern(t *testing.T) {
trueTestCases := []string{
"[[stuff]]",
"[[]]",
"[[stuff|title=Difficult name with spaces*!]]",
}
falseTestCases := []string{
"test",
"abcdefg",
"[[]",
"[[",
"[]",
"]]",
"abcdefghijklmnopqrstuvwxyz",
}
for _, testCase := range trueTestCases {
assert.True(t, markup.ShortLinkPattern.MatchString(testCase))
}
for _, testCase := range falseTestCases {
assert.False(t, markup.ShortLinkPattern.MatchString(testCase))
}
`<p><a href="`+result+`" rel="nofollow"><img src="`+result+`" title="`+title+`" alt="`+title+`"/></a></p>`)
}
func testAnswers(baseURLContent, baseURLImages string) []string {
@ -185,7 +97,7 @@ func testAnswers(baseURLContent, baseURLImages string) []string {
<ul>
<li>Bezier widget (by <a href="` + AppURL + `r-lyeh" rel="nofollow">@r-lyeh</a>) <a href="http://localhost:3000/ocornut/imgui/issues/786" rel="nofollow">#786</a></li>
<li>Node graph editors https://github.com/ocornut/imgui/issues/306</li>
<li>Node graph editors <a href="https://github.com/ocornut/imgui/issues/306" rel="nofollow">https://github.com/ocornut/imgui/issues/306</a></li>
<li><a href="` + baseURLContent + `/memory_editor_example" rel="nofollow">Memory Editor</a></li>
<li><a href="` + baseURLContent + `/plot_var_example" rel="nofollow">Plot var helper</a></li>
</ul>
@ -201,14 +113,14 @@ func testAnswers(baseURLContent, baseURLImages string) []string {
<table>
<thead>
<tr>
<th><a href="` + baseURLImages + `/images/icon-install.png" rel="nofollow"><img src="` + baseURLImages + `/images/icon-install.png" alt="images/icon-install.png" title="icon-install.png"/></a></th>
<th><a href="` + baseURLImages + `/images/icon-install.png" rel="nofollow"><img src="` + baseURLImages + `/images/icon-install.png" title="icon-install.png" alt="images/icon-install.png"/></a></th>
<th><a href="` + baseURLContent + `/Installation" rel="nofollow">Installation</a></th>
</tr>
</thead>
<tbody>
<tr>
<td><a href="` + baseURLImages + `/images/icon-usage.png" rel="nofollow"><img src="` + baseURLImages + `/images/icon-usage.png" alt="images/icon-usage.png" title="icon-usage.png"/></a></td>
<td><a href="` + baseURLImages + `/images/icon-usage.png" rel="nofollow"><img src="` + baseURLImages + `/images/icon-usage.png" title="icon-usage.png" alt="images/icon-usage.png"/></a></td>
<td><a href="` + baseURLContent + `/Usage" rel="nofollow">Usage</a></td>
</tr>
</tbody>
@ -218,9 +130,9 @@ func testAnswers(baseURLContent, baseURLImages string) []string {
<ol>
<li><a href="https://github.com/libgdx/libgdx/wiki/Gradle-on-the-Commandline#packaging-for-the-desktop" rel="nofollow">Package your libGDX application</a>
<a href="` + baseURLImages + `/images/1.png" rel="nofollow"><img src="` + baseURLImages + `/images/1.png" alt="images/1.png" title="1.png"/></a></li>
<a href="` + baseURLImages + `/images/1.png" rel="nofollow"><img src="` + baseURLImages + `/images/1.png" title="1.png" alt="images/1.png"/></a></li>
<li>Perform a test run by hitting the Run! button.
<a href="` + baseURLImages + `/images/2.png" rel="nofollow"><img src="` + baseURLImages + `/images/2.png" alt="images/2.png" title="2.png"/></a></li>
<a href="` + baseURLImages + `/images/2.png" rel="nofollow"><img src="` + baseURLImages + `/images/2.png" title="2.png" alt="images/2.png"/></a></li>
</ol>
`,
}

8
modules/markup/markup.go
View File

@ -7,6 +7,8 @@ package markup
import (
"path/filepath"
"strings"
"code.gitea.io/gitea/modules/log"
)
// Init initialize regexps for markdown parsing
@ -69,7 +71,11 @@ func RenderWiki(filename string, rawBytes []byte, urlPrefix string, metas map[st
func render(parser Parser, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
urlPrefix = strings.Replace(urlPrefix, " ", "+", -1)
result := parser.Render(rawBytes, urlPrefix, metas, isWiki)
result = PostProcess(result, urlPrefix, metas, isWiki)
// TODO: one day the error should be returned.
result, err := PostProcess(result, urlPrefix, metas, isWiki)
if err != nil {
log.Error(3, "PostProcess: %v", err)
}
return SanitizeBytes(result)
}

42
modules/templates/helper.go
View File

@ -10,6 +10,7 @@ import (
"encoding/json"
"errors"
"fmt"
"html"
"html/template"
"mime"
"net/url"
@ -27,7 +28,6 @@ import (
"golang.org/x/net/html/charset"
"golang.org/x/text/transform"
"gopkg.in/editorconfig/editorconfig-core-go.v1"
"html"
)
// NewFuncMap returns functions for injecting to templates
@ -280,26 +280,21 @@ func ReplaceLeft(s, old, new string) string {
// RenderCommitMessage renders commit message with XSS-safe and special links.
func RenderCommitMessage(msg, urlPrefix string, metas map[string]string) template.HTML {
return renderCommitMessage(msg, markup.RenderIssueIndexPatternOptions{
URLPrefix: urlPrefix,
Metas: metas,
})
return RenderCommitMessageLink(msg, urlPrefix, "", metas)
}
// RenderCommitMessageLink renders commit message as a XXS-safe link to the provided
// default url, handling for special links.
func RenderCommitMessageLink(msg, urlPrefix string, urlDefault string, metas map[string]string) template.HTML {
return renderCommitMessage(msg, markup.RenderIssueIndexPatternOptions{
DefaultURL: urlDefault,
URLPrefix: urlPrefix,
Metas: metas,
})
}
func renderCommitMessage(msg string, opts markup.RenderIssueIndexPatternOptions) template.HTML {
func RenderCommitMessageLink(msg, urlPrefix, urlDefault string, metas map[string]string) template.HTML {
cleanMsg := template.HTMLEscapeString(msg)
fullMessage := string(markup.RenderIssueIndexPattern([]byte(cleanMsg), opts))
msgLines := strings.Split(strings.TrimSpace(fullMessage), "\n")
// we can safely assume that it will not return any error, since there
// shouldn't be any special HTML.
fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, urlDefault, metas)
if err != nil {
log.Error(3, "RenderCommitMessage: %v", err)
return ""
}
msgLines := strings.Split(strings.TrimSpace(string(fullMessage)), "\n")
if len(msgLines) == 0 {
return template.HTML("")
}
@ -308,16 +303,13 @@ func renderCommitMessage(msg string, opts markup.RenderIssueIndexPatternOptions)
// RenderCommitBody extracts the body of a commit message without its title.
func RenderCommitBody(msg, urlPrefix string, metas map[string]string) template.HTML {
return renderCommitBody(msg, markup.RenderIssueIndexPatternOptions{
URLPrefix: urlPrefix,
Metas: metas,
})
}
func renderCommitBody(msg string, opts markup.RenderIssueIndexPatternOptions) template.HTML {
cleanMsg := template.HTMLEscapeString(msg)
fullMessage := string(markup.RenderIssueIndexPattern([]byte(cleanMsg), opts))
body := strings.Split(strings.TrimSpace(fullMessage), "\n")
fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, "", metas)
if err != nil {
log.Error(3, "RenderCommitMessage: %v", err)
return ""
}
body := strings.Split(strings.TrimSpace(string(fullMessage)), "\n")
if len(body) == 0 {
return template.HTML("")
}

2
public/js/index.js
View File

@ -771,7 +771,6 @@ function initWikiForm() {
function (data) {
preview.innerHTML = '<div class="markdown">' + data + '</div>';
emojify.run($('.editor-preview')[0]);
$('.editor-preview').autolink();
}
);
}, 0);
@ -1549,7 +1548,6 @@ $(document).ready(function () {
node.append('<a class="anchor" href="#' + name + '"><span class="octicon octicon-link"></span></a>');
});
});
$('.markdown').autolink();
$('.issue-checkbox').click(function() {
var numChecked = $('.issue-checkbox').children('input:checked').length;

21
public/vendor/plugins/autolink/LICENSE vendored
View File

@ -1,21 +0,0 @@
The MIT License (MIT)
Copyright (c) 2015 egoist 0x142857@gmail.com
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

45
public/vendor/plugins/autolink/autolink.js vendored
View File

@ -1,45 +0,0 @@
(function () {
var re = /((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:\/[\+~%\/\.\w\-]*)?\??(?:[\-\+:=&;%@\.\w]*)#?(?:[\.\!\/\\\w]*))?)/g;
function textNodesUnder(node) {
var textNodes = [];
if(typeof document.createTreeWalker === 'function') {
// Efficient TreeWalker
var currentNode, walker;
walker = document.createTreeWalker(node, NodeFilter.SHOW_TEXT, null, false);
while(currentNode = walker.nextNode()) {
textNodes.push(currentNode);
}
} else {
// Less efficient recursive function
for(node = node.firstChild; node; node = node.nextSibling) {
if(node.nodeType === 3) {
textNodes.push(node);
} else {
textNodes = textNodes.concat(textNodesUnder(node));
}
}
}
return textNodes;
}
function processNode(node) {
re.lastIndex = 0;
var results = re.exec(node.textContent);
if(results !== null) {
if($(node).parents().filter('code').length === 0) {
$(node).replaceWith(
$('<span />').html(
node.nodeValue.replace(re, '<a href="$1">$1</a>')
)
);
}
}
}
jQuery.fn.autolink = function () {
this.each(function () {
textNodesUnder(this).forEach(processNode);
});
return this;
};
})();

4
routers/api/v1/misc/markdown_test.go
View File

@ -73,7 +73,7 @@ func TestAPI_RenderGFM(t *testing.T) {
<ul>
<li><a href="` + AppSubURL + `wiki/Links" rel="nofollow">Links, Language bindings, Engine bindings</a></li>
<li><a href="` + AppSubURL + `wiki/Tips" rel="nofollow">Tips</a></li>
<li>Bezier widget (by <a href="` + AppURL + `r-lyeh" rel="nofollow">@r-lyeh</a>) https://github.com/ocornut/imgui/issues/786</li>
<li>Bezier widget (by <a href="` + AppURL + `r-lyeh" rel="nofollow">@r-lyeh</a>) <a href="https://github.com/ocornut/imgui/issues/786" rel="nofollow">https://github.com/ocornut/imgui/issues/786</a></li>
</ul>
`,
// wine-staging wiki home extract: special wiki syntax, images
@ -96,7 +96,7 @@ Here are some links to the most important topics. You can find the full list of
<p>Here are some links to the most important topics. You can find the full list of pages at the sidebar.</p>
<p><a href="` + AppSubURL + `wiki/Configuration" rel="nofollow">Configuration</a>
<a href="` + AppSubURL + `wiki/raw/images/icon-bug.png" rel="nofollow"><img src="` + AppSubURL + `wiki/raw/images/icon-bug.png" alt="images/icon-bug.png" title="icon-bug.png"/></a></p>
<a href="` + AppSubURL + `wiki/raw/images/icon-bug.png" rel="nofollow"><img src="` + AppSubURL + `wiki/raw/images/icon-bug.png" title="icon-bug.png" alt="images/icon-bug.png"/></a></p>
`,
// Guard wiki sidebar: special syntax
`[[Guardfile-DSL / Configuring-Guard|Guardfile-DSL---Configuring-Guard]]`,

1
templates/base/footer.tmpl
View File

@ -122,7 +122,6 @@
emojiTribute.attach(document.getElementById('content'))
</script>
{{end}}
<script src="{{AppSubUrl}}/vendor/plugins/autolink/autolink.js"></script>
<script src="{{AppSubUrl}}/vendor/plugins/emojify/emojify.min.js"></script>
<script src="{{AppSubUrl}}/vendor/plugins/clipboard/clipboard.min.js"></script>
<script src="{{AppSubUrl}}/vendor/plugins/vue/vue.min.js"></script>