Changelog for v1.15.7 (#17871)

* Changelog for v1.15.7

Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: 6543 <6543@obermui.de>

.golangci.yml

@@ -9,7 +9,6 @@ linters:
     - unused
     - structcheck
     - varcheck
-    - golint
     - dupl
     #- gocyclo # The cyclomatic complexety of a lot of functions is too high, we should refactor those another time.
     - gofmt

CHANGELOG.md

@@ -4,6 +4,65 @@ This changelog goes through all the changes that have been made in each release
 without substantial changes to our git log; to see the highlights of what has
 been added to each release, please refer to the [blog](https://blog.gitea.io).
 
+## [1.15.7](https://github.com/go-gitea/gitea/releases/tag/v1.15.7) - 2021-12-01
+
+* ENHANCEMENTS
+  * Only allow webhook to send requests to allowed hosts (#17482) (#17510)
+  * Fix login redirection links (#17451) (#17473)
+* BUGFIXES
+  * Fix database inconsistent when admin change user email (#17549) (#17840)
+  * Use correct user on releases (#17806) (#17818)
+  * Fix commit count in tag view (#17698) (#17790)
+  * Fix close issue but time watcher still running (#17643) (#17761)
+  * Fix Migrate Description (#17692) (#17727)
+  * Fix bug when project board get open issue number (#17703) (#17726)
+  * Return 400 but not 500 when request archive with wrong format (#17691) (#17700)
+  * Fix bug when read mysql database max lifetime (#17682) (#17690)
+  * Fix database deadlock when update issue labels (#17649) (#17665)
+  * Fix bug on detect issue/comment writer (#17592)
+  * Remove appSubUrl from pasted images (#17572) (#17588)
+  * Make `ParsePatch` more robust (#17573) (#17580)
+  * Fix stats upon searching issues (#17566) (#17578)
+  * Escape issue titles in comments list (#17555) (#17556)
+  * Fix zero created time bug on commit api (#17546) (#17547)
+  * Fix database keyword quote problem on migration v161 (#17522) (#17523)
+  * Fix email with + when active (#17518) (#17520)
+  * Stop double encoding blame commit messages (#17498) (#17500)
+  * Quote the table name in CountOrphanedObjects (#17487) (#17488)
+  * Run Migrate in Install rather than just SyncTables (#17475) (#17486)
+* BUILD
+  * Fix golangci-lint warnings (#17598 et al) (#17668)
+* MISC
+  * Preserve color when inverting emojis (#17797) (#17799)
+
+## [1.15.6](https://github.com/go-gitea/gitea/releases/tag/v1.15.6) - 2021-10-28
+
+* BUGFIXES
+  * Prevent panic in serv.go with Deploy Keys (#17434) (#17435)
+  * Fix CSV render error (#17406) (#17431)
+  * Read expected buffer size (#17409) (#17430)
+  * Ensure that restricted users can access repos for which they are members (#17460) (#17464)
+  * Make commit-statuses popup show correctly (#17447) (#17466)
+* TESTING
+  * Add integration tests for private.NoServCommand and private.ServCommand (#17456) (#17463)
+
+## [1.15.5](https://github.com/go-gitea/gitea/releases/tag/v1.15.5) - 2021-10-21
+
+* SECURITY
+  * Upgrade Bluemonday to v1.0.16 (#17372) (#17374)
+  * Ensure correct SSH permissions check for private and restricted users (#17370) (#17373)
+* BUGFIXES
+  * Prevent NPE in CSV diff rendering when column removed (#17018) (#17377)
+  * Offer rsa-sha2-512 and rsa-sha2-256 algorithms in internal SSH (#17281) (#17376)
+  * Don't panic if we fail to parse U2FRegistration data (#17304) (#17371)
+  * Ensure popup text is aligned left (backport for 1.15) (#17343)
+  * Ensure that git daemon export ok is created for mirrors (#17243) (#17306)
+  * Disable core.protectNTFS (#17300) (#17302)
+  * Use pointer for wrappedConn methods (#17295) (#17296)
+  * AutoRegistration is supposed to be working with disabled registration (backport) (#17292)
+  * Handle duplicate keys on GPG key ring (#17242) (#17284)
+  * Fix SVG side by side comparison link (#17375) (#17391)
+
 ## [1.15.4](https://github.com/go-gitea/gitea/releases/tag/v1.15.4) - 2021-10-08
 * BUGFIXES
   * Raw file API: don't try to interpret 40char filenames as commit SHA (#17185) (#17272)

build.go

@@ -2,7 +2,8 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
-//+build vendor
+//go:build vendor
+// +build vendor
 
 package main
 

build/generate-bindata.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build ignore
 // +build ignore
 
 package main

build/generate-emoji.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build ignore
 // +build ignore
 
 package main

build/generate-gitignores.go

@@ -1,3 +1,4 @@
+//go:build ignore
 // +build ignore
 
 package main

build/generate-licenses.go

@@ -1,3 +1,4 @@
+//go:build ignore
 // +build ignore
 
 package main

build/gocovmerge.go

@@ -6,6 +6,7 @@
 // gocovmerge takes the results from multiple `go test -coverprofile` runs and
 // merges them into one profile
 
+//go:build ignore
 // +build ignore
 
 package main

cmd/docs.go

@@ -43,7 +43,11 @@ func runDocs(ctx *cli.Context) error {
 		// Clean up markdown. The following bug was fixed in v2, but is present in v1.
 		// It affects markdown output (even though the issue is referring to man pages)
 		// https://github.com/urfave/cli/issues/1040
-		docs = docs[strings.Index(docs, "#"):]
+		firstHashtagIndex := strings.Index(docs, "#")
+
+		if firstHashtagIndex > 0 {
+			docs = docs[firstHashtagIndex:]
+		}
 	}
 
 	out := os.Stdout

cmd/embedded.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build bindata
 // +build bindata
 
 package cmd

cmd/embedded_stub.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !bindata
 // +build !bindata
 
 package cmd

cmd/web.go

@@ -194,6 +194,10 @@ func listen(m http.Handler, handleRedirector bool) error {
 		listenAddr = net.JoinHostPort(listenAddr, setting.HTTPPort)
 	}
 	log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL)
+	// This can be useful for users, many users do wrong to their config and get strange behaviors behind a reverse-proxy.
+	// A user may fix the configuration mistake when he sees this log.
+	// And this is also very helpful to maintainers to provide help to users to resolve their configuration problems.
+	log.Info("AppURL(ROOT_URL): %s", setting.AppURL)
 
 	if setting.LFS.StartServer {
 		log.Info("LFS server enabled")

custom/conf/app.example.ini

@@ -576,6 +576,8 @@ PATH =
 ;;
 ;; (Go-Git only) Don't cache objects greater than this in memory. (Set to 0 to disable.)
 ;LARGE_OBJECT_THRESHOLD = 1048576
+;; Set to true to forcibly set core.protectNTFS=false
+;DISABLE_CORE_PROTECT_NTFS=false
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1386,6 +1388,13 @@ PATH =
 ;; Deliver timeout in seconds
 ;DELIVER_TIMEOUT = 5
 ;;
+;; Webhook can only call allowed hosts for security reasons. Comma separated list, eg: external, 192.168.1.0/24, *.mydomain.com
+;; Built-in: loopback (for localhost), private (for LAN/intranet), external (for public hosts on internet), * (for all hosts)
+;; CIDR list: 1.2.3.0/8, 2001:db8::/32
+;; Wildcard hosts: *.mydomain.com, 192.168.100.*
+;; Default to * for 1.15.x, external for 1.16 and later
+;ALLOWED_HOST_LIST = *
+;;
 ;; Allow insecure certification
 ;SKIP_TLS_VERIFY = false
 ;;

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -545,6 +545,14 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type
 
 - `QUEUE_LENGTH`: **1000**: Hook task queue length. Use caution when editing this value.
 - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks.
+- `ALLOWED_HOST_LIST`: `*`: Default to `*` for 1.15.x, `external` for 1.16 and later. Webhook can only call allowed hosts for security reasons. Comma separated list.
+  - Built-in networks:
+    - `loopback`: 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included.
+    - `private`: RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet.
+    - `external`: A valid non-private unicast IP, you can access all hosts on public internet. 
+    - `*`: All hosts are allowed.
+  - CIDR list: `1.2.3.0/8` for IPv4 and `2001:db8::/32` for IPv6
+  - Wildcard hosts: `*.mydomain.com`, `192.168.100.*`
 - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification.
 - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page.
 - `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
@@ -839,6 +847,7 @@ NB: You must have `DISABLE_ROUTER_LOG` set to `false` for this option to take ef
 - `VERBOSE_PUSH`: **true**: Print status information about pushes as they are being processed.
 - `VERBOSE_PUSH_DELAY`: **5s**: Only print verbose information if push takes longer than this delay.
 - `LARGE_OBJECT_THRESHOLD`: **1048576**: (Go-Git only), don't cache objects greater than this in memory. (Set to 0 to disable.)
+- `DISABLE_CORE_PROTECT_NTFS`: **false** Set to true to forcibly set `core.protectNTFS` to false.
 ## Git - Timeout settings (`git.timeout`)
 - `DEFAUlT`: **360**: Git operations default timeout seconds.
 - `MIGRATE`: **600**: Migrate external repositories timeout seconds.

go.mod

@@ -80,7 +80,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/mattn/go-sqlite3 v1.14.8
 	github.com/mholt/archiver/v3 v3.5.0
-	github.com/microcosm-cc/bluemonday v1.0.15
+	github.com/microcosm-cc/bluemonday v1.0.16
 	github.com/miekg/dns v1.1.43 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/minio-go/v7 v7.0.12
@@ -125,7 +125,7 @@ require (
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.18.1 // indirect
 	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
+	golang.org/x/net v0.0.0-20211020060615-d418f374d309
 	golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
 	golang.org/x/text v0.3.6

go.sum

@@ -868,8 +868,8 @@ github.com/mholt/acmez v0.1.3 h1:J7MmNIk4Qf9b8mAGqAh4XkNeowv3f1zW816yf4zt7Qk=
 github.com/mholt/acmez v0.1.3/go.mod h1:8qnn8QA/Ewx8E3ZSsmscqsIjhhpxuy9vqdgbX2ceceM=
 github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE=
 github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc=
-github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY=
-github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30=
+github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc=
+github.com/microcosm-cc/bluemonday v1.0.16/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
@@ -1364,8 +1364,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
 golang.org/x/net v0.0.0-20210331060903-cb1fcc7394e5/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
+golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

integrations/api_private_serv_test.go

@@ -0,0 +1,154 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/private"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAPIPrivateNoServ(t *testing.T) {
+	onGiteaRun(t, func(*testing.T, *url.URL) {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		key, user, err := private.ServNoCommand(ctx, 1)
+		assert.NoError(t, err)
+		assert.Equal(t, int64(2), user.ID)
+		assert.Equal(t, "user2", user.Name)
+		assert.Equal(t, int64(1), key.ID)
+		assert.Equal(t, "user2@localhost", key.Name)
+
+		deployKey, err := models.AddDeployKey(1, "test-deploy", "sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBGXEEzWmm1dxb+57RoK5KVCL0w2eNv9cqJX2AGGVlkFsVDhOXHzsadS3LTK4VlEbbrDMJdoti9yM8vclA8IeRacAAAAEc3NoOg== nocomment", false)
+		assert.NoError(t, err)
+
+		key, user, err = private.ServNoCommand(ctx, deployKey.KeyID)
+		assert.NoError(t, err)
+		assert.Empty(t, user)
+		assert.Equal(t, deployKey.KeyID, key.ID)
+		assert.Equal(t, "test-deploy", key.Name)
+	})
+}
+
+func TestAPIPrivateServ(t *testing.T) {
+	onGiteaRun(t, func(*testing.T, *url.URL) {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// Can push to a repo we own
+		results, err := private.ServCommand(ctx, 1, "user2", "repo1", models.AccessModeWrite, "git-upload-pack", "")
+		assert.NoError(t, err)
+		assert.False(t, results.IsWiki)
+		assert.False(t, results.IsDeployKey)
+		assert.Equal(t, int64(1), results.KeyID)
+		assert.Equal(t, "user2@localhost", results.KeyName)
+		assert.Equal(t, "user2", results.UserName)
+		assert.Equal(t, int64(2), results.UserID)
+		assert.Equal(t, "user2", results.OwnerName)
+		assert.Equal(t, "repo1", results.RepoName)
+		assert.Equal(t, int64(1), results.RepoID)
+
+		// Cannot push to a private repo we're not associated with
+		results, err = private.ServCommand(ctx, 1, "user15", "big_test_private_1", models.AccessModeWrite, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Cannot pull from a private repo we're not associated with
+		results, err = private.ServCommand(ctx, 1, "user15", "big_test_private_1", models.AccessModeRead, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Can pull from a public repo we're not associated with
+		results, err = private.ServCommand(ctx, 1, "user15", "big_test_public_1", models.AccessModeRead, "git-upload-pack", "")
+		assert.NoError(t, err)
+		assert.False(t, results.IsWiki)
+		assert.False(t, results.IsDeployKey)
+		assert.Equal(t, int64(1), results.KeyID)
+		assert.Equal(t, "user2@localhost", results.KeyName)
+		assert.Equal(t, "user2", results.UserName)
+		assert.Equal(t, int64(2), results.UserID)
+		assert.Equal(t, "user15", results.OwnerName)
+		assert.Equal(t, "big_test_public_1", results.RepoName)
+		assert.Equal(t, int64(17), results.RepoID)
+
+		// Cannot push to a public repo we're not associated with
+		results, err = private.ServCommand(ctx, 1, "user15", "big_test_public_1", models.AccessModeWrite, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Add reading deploy key
+		deployKey, err := models.AddDeployKey(19, "test-deploy", "sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBGXEEzWmm1dxb+57RoK5KVCL0w2eNv9cqJX2AGGVlkFsVDhOXHzsadS3LTK4VlEbbrDMJdoti9yM8vclA8IeRacAAAAEc3NoOg== nocomment", true)
+		assert.NoError(t, err)
+
+		// Can pull from repo we're a deploy key for
+		results, err = private.ServCommand(ctx, deployKey.KeyID, "user15", "big_test_private_1", models.AccessModeRead, "git-upload-pack", "")
+		assert.NoError(t, err)
+		assert.False(t, results.IsWiki)
+		assert.True(t, results.IsDeployKey)
+		assert.Equal(t, deployKey.KeyID, results.KeyID)
+		assert.Equal(t, "test-deploy", results.KeyName)
+		assert.Equal(t, "user15", results.UserName)
+		assert.Equal(t, int64(15), results.UserID)
+		assert.Equal(t, "user15", results.OwnerName)
+		assert.Equal(t, "big_test_private_1", results.RepoName)
+		assert.Equal(t, int64(19), results.RepoID)
+
+		// Cannot push to a private repo with reading key
+		results, err = private.ServCommand(ctx, deployKey.KeyID, "user15", "big_test_private_1", models.AccessModeWrite, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Cannot pull from a private repo we're not associated with
+		results, err = private.ServCommand(ctx, deployKey.ID, "user15", "big_test_private_2", models.AccessModeRead, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Cannot pull from a public repo we're not associated with
+		results, err = private.ServCommand(ctx, deployKey.ID, "user15", "big_test_public_1", models.AccessModeRead, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Add writing deploy key
+		deployKey, err = models.AddDeployKey(20, "test-deploy", "sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBGXEEzWmm1dxb+57RoK5KVCL0w2eNv9cqJX2AGGVlkFsVDhOXHzsadS3LTK4VlEbbrDMJdoti9yM8vclA8IeRacAAAAEc3NoOg== nocomment", false)
+		assert.NoError(t, err)
+
+		// Cannot push to a private repo with reading key
+		results, err = private.ServCommand(ctx, deployKey.KeyID, "user15", "big_test_private_1", models.AccessModeWrite, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Can pull from repo we're a writing deploy key for
+		results, err = private.ServCommand(ctx, deployKey.KeyID, "user15", "big_test_private_2", models.AccessModeRead, "git-upload-pack", "")
+		assert.NoError(t, err)
+		assert.False(t, results.IsWiki)
+		assert.True(t, results.IsDeployKey)
+		assert.Equal(t, deployKey.KeyID, results.KeyID)
+		assert.Equal(t, "test-deploy", results.KeyName)
+		assert.Equal(t, "user15", results.UserName)
+		assert.Equal(t, int64(15), results.UserID)
+		assert.Equal(t, "user15", results.OwnerName)
+		assert.Equal(t, "big_test_private_2", results.RepoName)
+		assert.Equal(t, int64(20), results.RepoID)
+
+		// Can push to repo we're a writing deploy key for
+		results, err = private.ServCommand(ctx, deployKey.KeyID, "user15", "big_test_private_2", models.AccessModeWrite, "git-upload-pack", "")
+		assert.NoError(t, err)
+		assert.False(t, results.IsWiki)
+		assert.True(t, results.IsDeployKey)
+		assert.Equal(t, deployKey.KeyID, results.KeyID)
+		assert.Equal(t, "test-deploy", results.KeyName)
+		assert.Equal(t, "user15", results.UserName)
+		assert.Equal(t, int64(15), results.UserID)
+		assert.Equal(t, "user15", results.OwnerName)
+		assert.Equal(t, "big_test_private_2", results.RepoName)
+		assert.Equal(t, int64(20), results.RepoID)
+
+	})
+
+}

integrations/api_repo_archive_test.go

@@ -0,0 +1,44 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAPIDownloadArchive(t *testing.T) {
+	defer prepareTestEnv(t)()
+
+	repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
+	user2 := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
+	session := loginUser(t, user2.LowerName)
+	token := getTokenForLoggedInUser(t, session)
+
+	link, _ := url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/archive/master.zip", user2.Name, repo.Name))
+	link.RawQuery = url.Values{"token": {token}}.Encode()
+	resp := MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK)
+	bs, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 320, len(bs))
+
+	link, _ = url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/archive/master.tar.gz", user2.Name, repo.Name))
+	link.RawQuery = url.Values{"token": {token}}.Encode()
+	resp = MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK)
+	bs, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 266, len(bs))
+
+	link, _ = url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/archive/master", user2.Name, repo.Name))
+	link.RawQuery = url.Values{"token": {token}}.Encode()
+	MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusBadRequest)
+}

integrations/api_user_heatmap_test.go

@@ -8,8 +8,10 @@ import (
 	"fmt"
 	"net/http"
 	"testing"
+	"time"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/timeutil"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -20,6 +22,10 @@ func TestUserHeatmap(t *testing.T) {
 	normalUsername := "user2"
 	session := loginUser(t, adminUsername)
 
+	var fakeNow = time.Date(2011, 10, 20, 0, 0, 0, 0, time.Local)
+	timeutil.Set(fakeNow)
+	defer timeutil.Unset()
+
 	urlStr := fmt.Sprintf("/api/v1/users/%s/heatmap", normalUsername)
 	req := NewRequest(t, "GET", urlStr)
 	resp := session.MakeRequest(t, req, http.StatusOK)

integrations/org_test.go

@@ -5,10 +5,12 @@
 package integrations
 
 import (
+	"fmt"
 	"net/http"
 	"strings"
 	"testing"
 
+	api "code.gitea.io/gitea/modules/structs"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -110,3 +112,64 @@ func TestPrivateOrg(t *testing.T) {
 	req = NewRequest(t, "GET", "/privated_org/private_repo_on_private_org")
 	session.MakeRequest(t, req, http.StatusOK)
 }
+
+func TestOrgRestrictedUser(t *testing.T) {
+	defer prepareTestEnv(t)()
+
+	// privated_org is a private org who has id 23
+	orgName := "privated_org"
+
+	// public_repo_on_private_org is a public repo on privated_org
+	repoName := "public_repo_on_private_org"
+
+	// user29 is a restricted user who is not a member of the organization
+	restrictedUser := "user29"
+
+	// #17003 reports a bug whereby adding a restricted user to a read-only team doesn't work
+
+	// assert restrictedUser cannot see the org or the public repo
+	restrictedSession := loginUser(t, restrictedUser)
+	req := NewRequest(t, "GET", fmt.Sprintf("/%s", orgName))
+	restrictedSession.MakeRequest(t, req, http.StatusNotFound)
+
+	req = NewRequest(t, "GET", fmt.Sprintf("/%s/%s", orgName, repoName))
+	restrictedSession.MakeRequest(t, req, http.StatusNotFound)
+
+	// Therefore create a read-only team
+	adminSession := loginUser(t, "user1")
+	token := getTokenForLoggedInUser(t, adminSession)
+
+	teamToCreate := &api.CreateTeamOption{
+		Name:                    "codereader",
+		Description:             "Code Reader",
+		IncludesAllRepositories: true,
+		Permission:              "read",
+		Units:                   []string{"repo.code"},
+	}
+
+	req = NewRequestWithJSON(t, "POST",
+		fmt.Sprintf("/api/v1/orgs/%s/teams?token=%s", orgName, token), teamToCreate)
+
+	var apiTeam api.Team
+
+	resp := adminSession.MakeRequest(t, req, http.StatusCreated)
+	DecodeJSON(t, resp, &apiTeam)
+	checkTeamResponse(t, &apiTeam, teamToCreate.Name, teamToCreate.Description, teamToCreate.IncludesAllRepositories,
+		teamToCreate.Permission, teamToCreate.Units)
+	checkTeamBean(t, apiTeam.ID, teamToCreate.Name, teamToCreate.Description, teamToCreate.IncludesAllRepositories,
+		teamToCreate.Permission, teamToCreate.Units)
+	//teamID := apiTeam.ID
+
+	// Now we need to add the restricted user to the team
+	req = NewRequest(t, "PUT",
+		fmt.Sprintf("/api/v1/teams/%d/members/%s?token=%s", apiTeam.ID, restrictedUser, token))
+	_ = adminSession.MakeRequest(t, req, http.StatusNoContent)
+
+	// Now we need to check if the restrictedUser can access the repo
+	req = NewRequest(t, "GET", fmt.Sprintf("/%s", orgName))
+	restrictedSession.MakeRequest(t, req, http.StatusOK)
+
+	req = NewRequest(t, "GET", fmt.Sprintf("/%s/%s", orgName, repoName))
+	restrictedSession.MakeRequest(t, req, http.StatusOK)
+
+}

models/consistency.go

@@ -302,7 +302,7 @@ func DeleteOrphanedIssues() error {
 // CountOrphanedObjects count subjects with have no existing refobject anymore
 func CountOrphanedObjects(subject, refobject, joinCond string) (int64, error) {
 	return x.Table("`"+subject+"`").
-		Join("LEFT", refobject, joinCond).
+		Join("LEFT", "`"+refobject+"`", joinCond).
 		Where(builder.IsNull{"`" + refobject + "`.id"}).
 		Count("id")
 }

models/fixtures/repository.yml

@@ -568,7 +568,7 @@
 -
   id: 40
   owner_id: 23
-  owner_name: limited_org
+  owner_name: privated_org
   lower_name: public_repo_on_private_org
   name: public_repo_on_private_org
   is_private: false
@@ -581,7 +581,7 @@
 -
   id: 41
   owner_id: 23
-  owner_name: limited_org
+  owner_name: privated_org
   lower_name: private_repo_on_private_org
   name: private_repo_on_private_org
   is_private: true

models/gpg_key_add.go

@@ -99,6 +99,46 @@ func AddGPGKey(ownerID int64, content, token, signature string) ([]*GPGKey, erro
 		verified = true
 	}
 
+	if len(ekeys) > 1 {
+		id2key := map[string]*openpgp.Entity{}
+		newEKeys := make([]*openpgp.Entity, 0, len(ekeys))
+		for _, ekey := range ekeys {
+			id := ekey.PrimaryKey.KeyIdString()
+			if original, has := id2key[id]; has {
+				// Coalesce this with the other one
+				for _, subkey := range ekey.Subkeys {
+					if subkey.PublicKey == nil {
+						continue
+					}
+					found := false
+
+					for _, originalSubkey := range original.Subkeys {
+						if originalSubkey.PublicKey == nil {
+							continue
+						}
+						if originalSubkey.PublicKey.KeyId == subkey.PublicKey.KeyId {
+							found = true
+							break
+						}
+					}
+					if !found {
+						original.Subkeys = append(original.Subkeys, subkey)
+					}
+				}
+				for name, identity := range ekey.Identities {
+					if _, has := original.Identities[name]; has {
+						continue
+					}
+					original.Identities[name] = identity
+				}
+				continue
+			}
+			id2key[id] = ekey
+			newEKeys = append(newEKeys, ekey)
+		}
+		ekeys = newEKeys
+	}
+
 	for _, ekey := range ekeys {
 		// Key ID cannot be duplicated.
 		has, err := sess.Where("key_id=?", ekey.PrimaryKey.KeyIdString()).

models/issue.go

@@ -1517,12 +1517,12 @@ func GetIssueStats(opts *IssueStatsOptions) (*IssueStats, error) {
 func getIssueStatsChunk(opts *IssueStatsOptions, issueIDs []int64) (*IssueStats, error) {
 	stats := &IssueStats{}
 
-	countSession := func(opts *IssueStatsOptions) *xorm.Session {
+	countSession := func(opts *IssueStatsOptions, issueIDs []int64) *xorm.Session {
 		sess := x.
 			Where("issue.repo_id = ?", opts.RepoID)
 
-		if len(opts.IssueIDs) > 0 {
-			sess.In("issue.id", opts.IssueIDs)
+		if len(issueIDs) > 0 {
+			sess.In("issue.id", issueIDs)
 		}
 
 		if len(opts.Labels) > 0 && opts.Labels != "0" {
@@ -1572,13 +1572,13 @@ func getIssueStatsChunk(opts *IssueStatsOptions, issueIDs []int64) (*IssueStats,
 	}
 
 	var err error
-	stats.OpenCount, err = countSession(opts).
+	stats.OpenCount, err = countSession(opts, issueIDs).
 		And("issue.is_closed = ?", false).
 		Count(new(Issue))
 	if err != nil {
 		return stats, err
 	}
-	stats.ClosedCount, err = countSession(opts).
+	stats.ClosedCount, err = countSession(opts, issueIDs).
 		And("issue.is_closed = ?", true).
 		Count(new(Issue))
 	return stats, err

models/issue_stopwatch.go

@@ -13,6 +13,26 @@ import (
 	"xorm.io/xorm"
 )
 
+// ErrIssueStopwatchNotExist represents an error that stopwatch is not exist
+type ErrIssueStopwatchNotExist struct {
+	UserID  int64
+	IssueID int64
+}
+
+func (err ErrIssueStopwatchNotExist) Error() string {
+	return fmt.Sprintf("issue stopwatch doesn't exist[uid: %d, issue_id: %d", err.UserID, err.IssueID)
+}
+
+// ErrIssueStopwatchAlreadyExist represents an error that stopwatch is already exist
+type ErrIssueStopwatchAlreadyExist struct {
+	UserID  int64
+	IssueID int64
+}
+
+func (err ErrIssueStopwatchAlreadyExist) Error() string {
+	return fmt.Sprintf("issue stopwatch already exists[uid: %d, issue_id: %d", err.UserID, err.IssueID)
+}
+
 // Stopwatch represents a stopwatch for time tracking.
 type Stopwatch struct {
 	ID          int64              `xorm:"pk autoincr"`
@@ -74,91 +94,141 @@ func hasUserStopwatch(e Engine, userID int64) (exists bool, sw *Stopwatch, err e
 	return
 }
 
+// FinishIssueStopwatchIfPossible if stopwatch exist then finish it otherwise ignore
+func FinishIssueStopwatchIfPossible(user *User, issue *Issue) error {
+	_, exists, err := getStopwatch(x, user.ID, issue.ID)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return nil
+	}
+	return FinishIssueStopwatch(user, issue)
+}
+
 // CreateOrStopIssueStopwatch will create or remove a stopwatch and will log it into issue's timeline.
 func CreateOrStopIssueStopwatch(user *User, issue *Issue) error {
+	_, exists, err := getStopwatch(x, user.ID, issue.ID)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return FinishIssueStopwatch(user, issue)
+	}
+	return CreateIssueStopwatch(user, issue)
+}
+
+// FinishIssueStopwatch if stopwatch exist then finish it otherwise return an error
+func FinishIssueStopwatch(user *User, issue *Issue) error {
 	sess := x.NewSession()
 	defer sess.Close()
 	if err := sess.Begin(); err != nil {
 		return err
 	}
-	if err := createOrStopIssueStopwatch(sess, user, issue); err != nil {
+	if err := finishIssueStopwatch(sess, user, issue); err != nil {
 		return err
 	}
 	return sess.Commit()
 }
 
-func createOrStopIssueStopwatch(e *xorm.Session, user *User, issue *Issue) error {
+func finishIssueStopwatch(e *xorm.Session, user *User, issue *Issue) error {
 	sw, exists, err := getStopwatch(e, user.ID, issue.ID)
 	if err != nil {
 		return err
 	}
+	if !exists {
+		return ErrIssueStopwatchNotExist{
+			UserID:  user.ID,
+			IssueID: issue.ID,
+		}
+	}
+
+	// Create tracked time out of the time difference between start date and actual date
+	timediff := time.Now().Unix() - int64(sw.CreatedUnix)
+
+	// Create TrackedTime
+	tt := &TrackedTime{
+		Created: time.Now(),
+		IssueID: issue.ID,
+		UserID:  user.ID,
+		Time:    timediff,
+	}
+
+	if _, err := e.Insert(tt); err != nil {
+		return err
+	}
+
+	if err := issue.loadRepo(e); err != nil {
+		return err
+	}
+	if _, err := createComment(e, &CreateCommentOptions{
+		Doer:    user,
+		Issue:   issue,
+		Repo:    issue.Repo,
+		Content: SecToTime(timediff),
+		Type:    CommentTypeStopTracking,
+		TimeID:  tt.ID,
+	}); err != nil {
+		return err
+	}
+	_, err = e.Delete(sw)
+	return err
+}
+
+// CreateIssueStopwatch creates a stopwatch if not exist, otherwise return an error
+func CreateIssueStopwatch(user *User, issue *Issue) error {
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+	if err := createIssueStopwatch(sess, user, issue); err != nil {
+		return err
+	}
+	return sess.Commit()
+}
+
+func createIssueStopwatch(e *xorm.Session, user *User, issue *Issue) error {
 	if err := issue.loadRepo(e); err != nil {
 		return err
 	}
 
+	// if another stopwatch is running: stop it
+	exists, sw, err := hasUserStopwatch(e, user.ID)
+	if err != nil {
+		return err
+	}
 	if exists {
-		// Create tracked time out of the time difference between start date and actual date
-		timediff := time.Now().Unix() - int64(sw.CreatedUnix)
-
-		// Create TrackedTime
-		tt := &TrackedTime{
-			Created: time.Now(),
-			IssueID: issue.ID,
-			UserID:  user.ID,
-			Time:    timediff,
-		}
-
-		if _, err := e.Insert(tt); err != nil {
-			return err
-		}
-
-		if _, err := createComment(e, &CreateCommentOptions{
-			Doer:    user,
-			Issue:   issue,
-			Repo:    issue.Repo,
-			Content: SecToTime(timediff),
-			Type:    CommentTypeStopTracking,
-			TimeID:  tt.ID,
-		}); err != nil {
-			return err
-		}
-		if _, err := e.Delete(sw); err != nil {
-			return err
-		}
-	} else {
-		// if another stopwatch is running: stop it
-		exists, sw, err := hasUserStopwatch(e, user.ID)
+		issue, err := getIssueByID(e, sw.IssueID)
 		if err != nil {
 			return err
 		}
-		if exists {
-			issue, err := getIssueByID(e, sw.IssueID)
-			if err != nil {
-				return err
-			}
-			if err := createOrStopIssueStopwatch(e, user, issue); err != nil {
-				return err
-			}
-		}
-
-		// Create stopwatch
-		sw = &Stopwatch{
-			UserID:  user.ID,
-			IssueID: issue.ID,
-		}
-
-		if _, err := e.Insert(sw); err != nil {
+		if err := finishIssueStopwatch(e, user, issue); err != nil {
 			return err
 		}
+	}
 
-		if _, err := createComment(e, &CreateCommentOptions{
-			Doer:  user,
-			Issue: issue,
-			Repo:  issue.Repo,
-			Type:  CommentTypeStartTracking,
-		}); err != nil {
-			return err
-		}
+	// Create stopwatch
+	sw = &Stopwatch{
+		UserID:  user.ID,
+		IssueID: issue.ID,
+	}
+
+	if _, err := e.Insert(sw); err != nil {
+		return err
+	}
+
+	if err := issue.loadRepo(e); err != nil {
+		return err
+	}
+
+	if _, err := createComment(e, &CreateCommentOptions{
+		Doer:  user,
+		Issue: issue,
+		Repo:  issue.Repo,
+		Type:  CommentTypeStartTracking,
+	}); err != nil {
+		return err
 	}
 	return nil
 }

models/issue_test.go

@@ -5,7 +5,9 @@
 package models
 
 import (
+	"fmt"
 	"sort"
+	"sync"
 	"testing"
 	"time"
 
@@ -417,3 +419,43 @@ func TestIssue_ResolveMentions(t *testing.T) {
 	// Private repo, whole team
 	testSuccess("user17", "big_test_private_4", "user15", []string{"user17/owners"}, []int64{18})
 }
+
+func TestCorrectIssueStats(t *testing.T) {
+	assert.NoError(t, PrepareTestDatabase())
+
+	// Because the condition is to have chunked database look-ups,
+	// We have to more issues than `maxQueryParameters`, we will insert.
+	// maxQueryParameters + 10 issues into the testDatabase.
+	// Each new issues will have a constant description "Bugs are nasty"
+	// Which will be used later on.
+
+	issueAmount := maxQueryParameters + 10
+
+	var wg sync.WaitGroup
+	for i := 0; i < issueAmount; i++ {
+		wg.Add(1)
+		go func(i int) {
+			testInsertIssue(t, fmt.Sprintf("Issue %d", i+1), "Bugs are nasty", 0)
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+
+	// Now we will get all issueID's that match the "Bugs are nasty" query.
+	total, ids, err := SearchIssueIDsByKeyword("Bugs are nasty", []int64{1}, issueAmount, 0)
+
+	// Just to be sure.
+	assert.NoError(t, err)
+	assert.EqualValues(t, issueAmount, total)
+
+	// Now we will call the GetIssueStats with these IDs and if working,
+	// get the correct stats back.
+	issueStats, err := GetIssueStats(&IssueStatsOptions{
+		RepoID:   1,
+		IssueIDs: ids,
+	})
+
+	// Now check the values.
+	assert.NoError(t, err)
+	assert.EqualValues(t, issueStats.OpenCount, issueAmount)
+}

models/migrations/migrations.go

@@ -7,6 +7,7 @@ package migrations
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"reflect"
@@ -762,8 +763,14 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
 		}
 		tableSQL := string(res[0]["sql"])
 
+		// Get the string offset for column definitions: `CREATE TABLE ( column-definitions... )`
+		columnDefinitionsIndex := strings.Index(tableSQL, "(")
+		if columnDefinitionsIndex < 0 {
+			return errors.New("couldn't find column definitions")
+		}
+
 		// Separate out the column definitions
-		tableSQL = tableSQL[strings.Index(tableSQL, "("):]
+		tableSQL = tableSQL[columnDefinitionsIndex:]
 
 		// Remove the required columnNames
 		for _, name := range columnNames {

models/migrations/v161.go

@@ -5,6 +5,8 @@
 package migrations
 
 import (
+	"context"
+
 	"xorm.io/xorm"
 )
 
@@ -40,8 +42,17 @@ func convertTaskTypeToString(x *xorm.Engine) error {
 		return err
 	}
 
+	// to keep the migration could be rerun
+	exist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "hook_task", "type")
+	if err != nil {
+		return err
+	}
+	if !exist {
+		return nil
+	}
+
 	for i, s := range hookTaskTypes {
-		if _, err := x.Exec("UPDATE hook_task set typ = ? where type=?", s, i); err != nil {
+		if _, err := x.Exec("UPDATE hook_task set typ = ? where `type`=?", s, i); err != nil {
 			return err
 		}
 	}

models/migrations/v184.go

@@ -5,6 +5,7 @@
 package migrations
 
 import (
+	"context"
 	"fmt"
 
 	"code.gitea.io/gitea/modules/setting"
@@ -19,6 +20,22 @@ func renameTaskErrorsToMessage(x *xorm.Engine) error {
 		Status int `xorm:"index"`
 	}
 
+	// This migration maybe rerun so that we should check if it has been run
+	messageExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "task", "message")
+	if err != nil {
+		return err
+	}
+
+	if messageExist {
+		errorsExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "task", "errors")
+		if err != nil {
+			return err
+		}
+		if !errorsExist {
+			return nil
+		}
+	}
+
 	sess := x.NewSession()
 	defer sess.Close()
 	if err := sess.Begin(); err != nil {
@@ -29,6 +46,13 @@ func renameTaskErrorsToMessage(x *xorm.Engine) error {
 		return fmt.Errorf("error on Sync2: %v", err)
 	}
 
+	if messageExist {
+		// if both errors and message exist, drop message at first
+		if err := dropTableColumns(sess, "task", "message"); err != nil {
+			return err
+		}
+	}
+
 	switch {
 	case setting.Database.UseMySQL:
 		if _, err := sess.Exec("ALTER TABLE `task` CHANGE errors message text"); err != nil {

models/models.go

@@ -179,16 +179,35 @@ func syncTables() error {
 	return x.StoreEngine("InnoDB").Sync2(tables...)
 }
 
-// NewTestEngine sets a new test xorm.Engine
-func NewTestEngine() (err error) {
+// NewInstallTestEngine creates a new xorm.Engine for testing during install
+//
+// This function will cause the basic database schema to be created
+func NewInstallTestEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) {
 	x, err = GetNewEngine()
 	if err != nil {
-		return fmt.Errorf("Connect to database: %v", err)
+		return fmt.Errorf("failed to connect to database: %w", err)
 	}
 
 	x.SetMapper(names.GonicMapper{})
 	x.SetLogger(NewXORMLogger(!setting.IsProd()))
 	x.ShowSQL(!setting.IsProd())
+
+	x.SetDefaultContext(ctx)
+
+	if err = x.Ping(); err != nil {
+		return err
+	}
+
+	// We have to run migrateFunc here in case the user is re-running installation on a previously created DB.
+	// If we do not then table schemas will be changed and there will be conflicts when the migrations run properly.
+	//
+	// Installation should only be being re-run if users want to recover an old database.
+	// However, we should think carefully about should we support re-install on an installed instance,
+	// as there may be other problems due to secret reinitialization.
+	if err = migrateFunc(x); err != nil {
+		return fmt.Errorf("migrate: %v", err)
+	}
+
 	return syncTables()
 }