Add Allow-/Block-List for Migrate & Mirrors (#13610) (#13776)

* add black list and white list support for migrating repositories

* specify log message

* use blocklist/allowlist

* allways use lowercase to match url

* Apply allow/block

* Settings: use existing "migrations" section

* convert domains lower case

* dont store unused value

* Block private addresses for migration by default

* use proposed-upstream func to detect private IP addr

* add own error for blocked migration, add tests, imprufe api

* fix test

* fix-if-localhost-is-ipv4

* rename error & error message

* rename setting options

* Apply suggestions from code review

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

modules/matchlist/matchlist.go

@@ -0,0 +1,46 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package matchlist
+
+import (
+	"strings"
+
+	"github.com/gobwas/glob"
+)
+
+// Matchlist represents a block or allow list
+type Matchlist struct {
+	ruleGlobs []glob.Glob
+}
+
+// NewMatchlist creates a new block or allow list
+func NewMatchlist(rules ...string) (*Matchlist, error) {
+	for i := range rules {
+		rules[i] = strings.ToLower(rules[i])
+	}
+	list := Matchlist{
+		ruleGlobs: make([]glob.Glob, 0, len(rules)),
+	}
+
+	for _, rule := range rules {
+		rg, err := glob.Compile(rule)
+		if err != nil {
+			return nil, err
+		}
+		list.ruleGlobs = append(list.ruleGlobs, rg)
+	}
+
+	return &list, nil
+}
+
+// Match will matches
+func (b *Matchlist) Match(u string) bool {
+	for _, r := range b.ruleGlobs {
+		if r.Match(u) {
+			return true
+		}
+	}
+	return false
+}

modules/migrations/migrate.go

@@ -8,9 +8,13 @@ package migrations
 import (
 	"context"
 	"fmt"
+	"net"
+	"net/url"
+	"strings"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/matchlist"
 	"code.gitea.io/gitea/modules/migrations/base"
 	"code.gitea.io/gitea/modules/setting"
 )
@@ -20,6 +24,9 @@ type MigrateOptions = base.MigrateOptions
 
 var (
 	factories []base.DownloaderFactory
+
+	allowList *matchlist.Matchlist
+	blockList *matchlist.Matchlist
 )
 
 // RegisterDownloaderFactory registers a downloader factory
@@ -27,12 +34,49 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) {
 	factories = append(factories, factory)
 }
 
+func isMigrateURLAllowed(remoteURL string) error {
+	u, err := url.Parse(strings.ToLower(remoteURL))
+	if err != nil {
+		return err
+	}
+
+	if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") {
+		if len(setting.Migrations.AllowedDomains) > 0 {
+			if !allowList.Match(u.Host) {
+				return &models.ErrMigrationNotAllowed{Host: u.Host}
+			}
+		} else {
+			if blockList.Match(u.Host) {
+				return &models.ErrMigrationNotAllowed{Host: u.Host}
+			}
+		}
+	}
+
+	if !setting.Migrations.AllowLocalNetworks {
+		addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0])
+		if err != nil {
+			return &models.ErrMigrationNotAllowed{Host: u.Host, NotResolvedIP: true}
+		}
+		for _, addr := range addrList {
+			if isIPPrivate(addr) || !addr.IsGlobalUnicast() {
+				return &models.ErrMigrationNotAllowed{Host: u.Host, PrivateNet: addr.String()}
+			}
+		}
+	}
+
+	return nil
+}
+
 // MigrateRepository migrate repository according MigrateOptions
 func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
+	err := isMigrateURLAllowed(opts.CloneAddr)
+	if err != nil {
+		return nil, err
+	}
+
 	var (
 		downloader base.Downloader
 		uploader   = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
-		err        error
 	)
 
 	for _, factory := range factories {
@@ -308,3 +352,32 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	return nil
 }
+
+// Init migrations service
+func Init() error {
+	var err error
+	allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...)
+	if err != nil {
+		return fmt.Errorf("init migration allowList domains failed: %v", err)
+	}
+
+	blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...)
+	if err != nil {
+		return fmt.Errorf("init migration blockList domains failed: %v", err)
+	}
+
+	return nil
+}
+
+// isIPPrivate reports whether ip is a private address, according to
+// RFC 1918 (IPv4 addresses) and RFC 4193 (IPv6 addresses).
+// from https://github.com/golang/go/pull/42793
+// TODO remove if https://github.com/golang/go/issues/29146 got resolved
+func isIPPrivate(ip net.IP) bool {
+	if ip4 := ip.To4(); ip4 != nil {
+		return ip4[0] == 10 ||
+			(ip4[0] == 172 && ip4[1]&0xf0 == 16) ||
+			(ip4[0] == 192 && ip4[1] == 168)
+	}
+	return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc
+}

modules/migrations/migrate_test.go

@@ -0,0 +1,34 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMigrateWhiteBlocklist(t *testing.T) {
+	setting.Migrations.AllowedDomains = []string{"github.com"}
+	assert.NoError(t, Init())
+
+	err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git")
+	assert.Error(t, err)
+
+	err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git")
+	assert.NoError(t, err)
+
+	setting.Migrations.AllowedDomains = []string{}
+	setting.Migrations.BlockedDomains = []string{"github.com"}
+	assert.NoError(t, Init())
+
+	err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git")
+	assert.NoError(t, err)
+
+	err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git")
+	assert.Error(t, err)
+}

modules/setting/migrations.go

@@ -4,11 +4,18 @@
 
 package setting
 
+import (
+	"strings"
+)
+
 var (
 	// Migrations settings
 	Migrations = struct {
-		MaxAttempts  int
-		RetryBackoff int
+		MaxAttempts        int
+		RetryBackoff       int
+		AllowedDomains     []string
+		BlockedDomains     []string
+		AllowLocalNetworks bool
 	}{
 		MaxAttempts:  3,
 		RetryBackoff: 3,
@@ -19,4 +26,15 @@ func newMigrationsService() {
 	sec := Cfg.Section("migrations")
 	Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts)
 	Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff)
+
+	Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").Strings(",")
+	for i := range Migrations.AllowedDomains {
+		Migrations.AllowedDomains[i] = strings.ToLower(Migrations.AllowedDomains[i])
+	}
+	Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").Strings(",")
+	for i := range Migrations.BlockedDomains {
+		Migrations.BlockedDomains[i] = strings.ToLower(Migrations.BlockedDomains[i])
+	}
+
+	Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
 }