Prometheus endpoint (#5256)

* Add prometheus collector and route

* dep ensure -add github.com/prometheus/client_golang/prometheus

* dep ensure -update github.com/golang/protobuf

* add metrics to reserved usernames

* add comment head in metrics package

* fix style imports

* add metrics settings

* add bearer token check

* mapping metrics configs

* fix lint

* update config cheat sheet

* update conf sample, typo fix

Gopkg.lock

@@ -81,6 +81,14 @@
   pruneopts = "NUT"
   revision = "349dd0209470eabd9514242c688c403c0926d266"
 
+[[projects]]
+  branch = "master"
+  digest = "1:707ebe952a8b3d00b343c01536c79c73771d100f63ec6babeaed5c79e2b8a8dd"
+  name = "github.com/beorn7/perks"
+  packages = ["quantile"]
+  pruneopts = "NUT"
+  revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
+
 [[projects]]
   digest = "1:67351095005f164e748a5a21899d1403b03878cb2d40a7b0f742376e6eeda974"
   name = "github.com/blevesearch/bleve"
@@ -405,11 +413,12 @@
   revision = "7f3990acf1833faa5ebd0e86f0a4c72a4b5eba3c"
 
 [[projects]]
-  digest = "1:b64f9be717fdab5f75122dc3868e8ca9d003779b6bc55f64f39a0cddc698bf88"
+  digest = "1:97df918963298c287643883209a2c3f642e6593379f97ab400c2a2e219ab647d"
   name = "github.com/golang/protobuf"
   packages = ["proto"]
   pruneopts = "NUT"
-  revision = "99511271042a09d1e01baea8781caa5210fec66e"
+  revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
+  version = "v1.2.0"
 
 [[projects]]
   digest = "1:60e25fc5f5cfd7783f985ca99b4383e848981dddf0be584db7d809be20848e25"
@@ -573,6 +582,14 @@
   pruneopts = "NUT"
   revision = "c7c4067b79cc51e6dfdcef5c702e74b1e0fa7c75"
 
+[[projects]]
+  digest = "1:5985ef4caf91ece5d54817c11ea25f182697534f8ae6521eadcd628c142ac4b6"
+  name = "github.com/matttproud/golang_protobuf_extensions"
+  packages = ["pbutil"]
+  pruneopts = "NUT"
+  revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
+  version = "v1.0.1"
+
 [[projects]]
   branch = "master"
   digest = "1:1ed0f3c066eb9d1c2ff7a864a6fa595c70b9b49049cc46af6a6f7ff0e4655321"
@@ -644,6 +661,51 @@
   pruneopts = "NUT"
   revision = "54653902c20e47f3417541d35435cb6d6162e28a"
 
+[[projects]]
+  digest = "1:aa2da1df3327c3a338bb42f978407c07de74cd0a5bef35e9411881dffd444214"
+  name = "github.com/prometheus/client_golang"
+  packages = [
+    "prometheus",
+    "prometheus/internal",
+    "prometheus/promhttp",
+  ]
+  pruneopts = "NUT"
+  revision = "1cafe34db7fdec6022e17e00e1c1ea501022f3e4"
+  version = "v0.9.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4"
+  name = "github.com/prometheus/client_model"
+  packages = ["go"]
+  pruneopts = "NUT"
+  revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f"
+
+[[projects]]
+  branch = "master"
+  digest = "1:06375f3b602de9c99fa99b8484f0e949fd5273e6e9c6592b5a0dd4cd9085f3ea"
+  name = "github.com/prometheus/common"
+  packages = [
+    "expfmt",
+    "internal/bitbucket.org/ww/goautoneg",
+    "model",
+  ]
+  pruneopts = "NUT"
+  revision = "7e9e6cabbd393fc208072eedef99188d0ce788b6"
+
+[[projects]]
+  branch = "master"
+  digest = "1:102dea0c03a915acfc634b7c67f2662012b5483b56d9025e33f5188e112759b6"
+  name = "github.com/prometheus/procfs"
+  packages = [
+    ".",
+    "internal/util",
+    "nfs",
+    "xfs",
+  ]
+  pruneopts = "NUT"
+  revision = "185b4288413d2a0dd0806f78c90dde719829e5ae"
+
 [[projects]]
   branch = "master"
   digest = "1:5be01c22bc1040e2f6ce4755d51a0ac9cef823a9f2004fb1f9896a414ef519e6"
@@ -981,6 +1043,8 @@
     "github.com/nfnt/resize",
     "github.com/pquerna/otp",
     "github.com/pquerna/otp/totp",
+    "github.com/prometheus/client_golang/prometheus",
+    "github.com/prometheus/client_golang/prometheus/promhttp",
     "github.com/russross/blackfriday",
     "github.com/satori/go.uuid",
     "github.com/sergi/go-diff/diffmatchpatch",

Gopkg.toml

@@ -107,3 +107,7 @@ ignored = ["google.golang.org/appengine*"]
 [[override]]
   revision = "c10ba270aa0bf8b8c1c986e103859c67a9103061"
   name = "golang.org/x/oauth2"
+
+[[constraint]]
+  name = "github.com/prometheus/client_golang"
+  version = "0.9.0"

custom/conf/app.ini.sample

@@ -659,3 +659,9 @@ FILE_EXTENSIONS = .adoc,.asciidoc
 RENDER_COMMAND = "asciidoc --out-file=- -"
 ; Don't pass the file on STDIN, pass the filename as argument instead.
 IS_INPUT_FILE = false
+
+[metrics]
+; Enables metrics endpoint. True or false; default is false.
+ENABLED = false
+; If you want to add authorization, specify a token here
+TOKEN =

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -302,6 +302,11 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `PULL`: **300**: Git pull from internal repositories timeout seconds.
 - `GC`: **60**: Git repository GC timeout seconds.
 
+## Metrics (`metrics`)
+
+- `ENABLED`: **false**: Enables /metrics endpoint for prometheus. 
+- `TOKEN`: **\<empty\>**: You need to specify the token, if you want to include in the authorization the metrics . The same token need to be used in prometheus parameters `bearer_token` or `bearer_token_file`.
+
 ## API (`api`)
  
 - `ENABLE_SWAGGER_ENDPOINT`: **true**: Enables /api/swagger, /api/v1/swagger etc. endpoints. True or false; default is true. 

models/user.go

@@ -698,6 +698,7 @@ var (
 		"issues",
 		"js",
 		"less",
+		"metrics",
 		"new",
 		"org",
 		"plugins",

modules/metrics/collector.go

@@ -0,0 +1,299 @@
+// Copyright 2018 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package metrics
+
+import (
+	"code.gitea.io/gitea/models"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const namespace = "gitea_"
+
+// Collector implements the prometheus.Collector interface and
+// exposes gitea metrics for prometheus
+type Collector struct {
+	Accesses      *prometheus.Desc
+	Actions       *prometheus.Desc
+	Attachments   *prometheus.Desc
+	Comments      *prometheus.Desc
+	Follows       *prometheus.Desc
+	HookTasks     *prometheus.Desc
+	Issues        *prometheus.Desc
+	Labels        *prometheus.Desc
+	LoginSources  *prometheus.Desc
+	Milestones    *prometheus.Desc
+	Mirrors       *prometheus.Desc
+	Oauths        *prometheus.Desc
+	Organizations *prometheus.Desc
+	PublicKeys    *prometheus.Desc
+	Releases      *prometheus.Desc
+	Repositories  *prometheus.Desc
+	Stars         *prometheus.Desc
+	Teams         *prometheus.Desc
+	UpdateTasks   *prometheus.Desc
+	Users         *prometheus.Desc
+	Watches       *prometheus.Desc
+	Webhooks      *prometheus.Desc
+}
+
+// NewCollector returns a new Collector with all prometheus.Desc initialized
+func NewCollector() Collector {
+	return Collector{
+		Accesses: prometheus.NewDesc(
+			namespace+"accesses",
+			"Number of Accesses",
+			nil, nil,
+		),
+		Actions: prometheus.NewDesc(
+			namespace+"actions",
+			"Number of Actions",
+			nil, nil,
+		),
+		Attachments: prometheus.NewDesc(
+			namespace+"attachments",
+			"Number of Attachments",
+			nil, nil,
+		),
+		Comments: prometheus.NewDesc(
+			namespace+"comments",
+			"Number of Comments",
+			nil, nil,
+		),
+		Follows: prometheus.NewDesc(
+			namespace+"follows",
+			"Number of Follows",
+			nil, nil,
+		),
+		HookTasks: prometheus.NewDesc(
+			namespace+"hooktasks",
+			"Number of HookTasks",
+			nil, nil,
+		),
+		Issues: prometheus.NewDesc(
+			namespace+"issues",
+			"Number of Issues",
+			nil, nil,
+		),
+		Labels: prometheus.NewDesc(
+			namespace+"labels",
+			"Number of Labels",
+			nil, nil,
+		),
+		LoginSources: prometheus.NewDesc(
+			namespace+"loginsources",
+			"Number of LoginSources",
+			nil, nil,
+		),
+		Milestones: prometheus.NewDesc(
+			namespace+"milestones",
+			"Number of Milestones",
+			nil, nil,
+		),
+		Mirrors: prometheus.NewDesc(
+			namespace+"mirrors",
+			"Number of Mirrors",
+			nil, nil,
+		),
+		Oauths: prometheus.NewDesc(
+			namespace+"oauths",
+			"Number of Oauths",
+			nil, nil,
+		),
+		Organizations: prometheus.NewDesc(
+			namespace+"organizations",
+			"Number of Organizations",
+			nil, nil,
+		),
+		PublicKeys: prometheus.NewDesc(
+			namespace+"publickeys",
+			"Number of PublicKeys",
+			nil, nil,
+		),
+		Releases: prometheus.NewDesc(
+			namespace+"releases",
+			"Number of Releases",
+			nil, nil,
+		),
+		Repositories: prometheus.NewDesc(
+			namespace+"repositories",
+			"Number of Repositories",
+			nil, nil,
+		),
+		Stars: prometheus.NewDesc(
+			namespace+"stars",
+			"Number of Stars",
+			nil, nil,
+		),
+		Teams: prometheus.NewDesc(
+			namespace+"teams",
+			"Number of Teams",
+			nil, nil,
+		),
+		UpdateTasks: prometheus.NewDesc(
+			namespace+"updatetasks",
+			"Number of UpdateTasks",
+			nil, nil,
+		),
+		Users: prometheus.NewDesc(
+			namespace+"users",
+			"Number of Users",
+			nil, nil,
+		),
+		Watches: prometheus.NewDesc(
+			namespace+"watches",
+			"Number of Watches",
+			nil, nil,
+		),
+		Webhooks: prometheus.NewDesc(
+			namespace+"webhooks",
+			"Number of Webhooks",
+			nil, nil,
+		),
+	}
+
+}
+
+// Describe returns all possible prometheus.Desc
+func (c Collector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.Accesses
+	ch <- c.Actions
+	ch <- c.Attachments
+	ch <- c.Comments
+	ch <- c.Follows
+	ch <- c.HookTasks
+	ch <- c.Issues
+	ch <- c.Labels
+	ch <- c.LoginSources
+	ch <- c.Milestones
+	ch <- c.Mirrors
+	ch <- c.Oauths
+	ch <- c.Organizations
+	ch <- c.PublicKeys
+	ch <- c.Releases
+	ch <- c.Repositories
+	ch <- c.Stars
+	ch <- c.Teams
+	ch <- c.UpdateTasks
+	ch <- c.Users
+	ch <- c.Watches
+	ch <- c.Webhooks
+}
+
+// Collect returns the metrics with values
+func (c Collector) Collect(ch chan<- prometheus.Metric) {
+	stats := models.GetStatistic()
+
+	ch <- prometheus.MustNewConstMetric(
+		c.Accesses,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Access),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Actions,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Action),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Attachments,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Attachment),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Comments,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Comment),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Follows,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Follow),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.HookTasks,
+		prometheus.GaugeValue,
+		float64(stats.Counter.HookTask),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Issues,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Issue),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Labels,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Label),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.LoginSources,
+		prometheus.GaugeValue,
+		float64(stats.Counter.LoginSource),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Milestones,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Milestone),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Mirrors,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Mirror),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Oauths,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Oauth),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Organizations,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Org),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.PublicKeys,
+		prometheus.GaugeValue,
+		float64(stats.Counter.PublicKey),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Releases,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Release),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Repositories,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Repo),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Stars,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Star),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Teams,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Team),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.UpdateTasks,
+		prometheus.GaugeValue,
+		float64(stats.Counter.UpdateTask),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Users,
+		prometheus.GaugeValue,
+		float64(stats.Counter.User),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Watches,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Watch),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.Webhooks,
+		prometheus.GaugeValue,
+		float64(stats.Counter.Webhook),
+	)
+}

modules/setting/setting.go

@@ -561,6 +561,15 @@ var (
 		TrustedFacets []string
 	}{}
 
+	// Metrics settings
+	Metrics = struct {
+		Enabled bool
+		Token   string
+	}{
+		Enabled: false,
+		Token:   "",
+	}
+
 	// I18n settings
 	Langs     []string
 	Names     []string
@@ -1125,6 +1134,8 @@ func NewContext() {
 		log.Fatal(4, "Failed to map Git settings: %v", err)
 	} else if err = Cfg.Section("api").MapTo(&API); err != nil {
 		log.Fatal(4, "Failed to map API settings: %v", err)
+	} else if err = Cfg.Section("metrics").MapTo(&Metrics); err != nil {
+		log.Fatal(4, "Failed to map Metrics settings: %v", err)
 	}
 
 	sec = Cfg.Section("mirror")

routers/metrics.go

@@ -0,0 +1,30 @@
+// Copyright 2018 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package routers
+
+import (
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// Metrics validate auth token and render prometheus metrics
+func Metrics(ctx *context.Context) {
+	if setting.Metrics.Token == "" {
+		promhttp.Handler().ServeHTTP(ctx.Resp, ctx.Req.Request)
+		return
+	}
+	header := ctx.Header().Get("Authorization")
+	if header == "" {
+		ctx.Error(401)
+		return
+	}
+	if header != "Bearer "+setting.Metrics.Token {
+		ctx.Error(401)
+		return
+	}
+	promhttp.Handler().ServeHTTP(ctx.Resp, ctx.Req.Request)
+}

routers/routes/routes.go

@@ -16,6 +16,7 @@ import (
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/metrics"
 	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/public"
 	"code.gitea.io/gitea/modules/setting"
@@ -39,6 +40,7 @@ import (
 	"github.com/go-macaron/i18n"
 	"github.com/go-macaron/session"
 	"github.com/go-macaron/toolbox"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/tstranex/u2f"
 	"gopkg.in/macaron.v1"
 )
@@ -788,6 +790,14 @@ func RegisterRoutes(m *macaron.Macaron) {
 		}
 	})
 
+	// prometheus metrics endpoint
+	if setting.Metrics.Enabled {
+		c := metrics.NewCollector()
+		prometheus.MustRegister(c)
+
+		m.Get("/metrics", routers.Metrics)
+	}
+
 	// Not found handler.
 	m.NotFound(routers.NotFound)
 }

vendor/github.com/beorn7/perks/LICENSE

@@ -0,0 +1,20 @@
+Copyright (C) 2013 Blake Mizerany
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/golang/protobuf/LICENSE

@@ -1,7 +1,4 @@
-Go support for Protocol Buffers - Google's data interchange format
-
 Copyright 2010 The Go Authors.  All rights reserved.
-https://github.com/golang/protobuf
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are

vendor/github.com/golang/protobuf/proto/clone.go

@@ -35,22 +35,39 @@
 package proto
 
 import (
+	"fmt"
 	"log"
 	"reflect"
 	"strings"
 )
 
 // Clone returns a deep copy of a protocol buffer.
-func Clone(pb Message) Message {
-	in := reflect.ValueOf(pb)
+func Clone(src Message) Message {
+	in := reflect.ValueOf(src)
 	if in.IsNil() {
-		return pb
+		return src
 	}
-
 	out := reflect.New(in.Type().Elem())
-	// out is empty so a merge is a deep copy.
-	mergeStruct(out.Elem(), in.Elem())
-	return out.Interface().(Message)
+	dst := out.Interface().(Message)
+	Merge(dst, src)
+	return dst
+}
+
+// Merger is the interface representing objects that can merge messages of the same type.
+type Merger interface {
+	// Merge merges src into this message.
+	// Required and optional fields that are set in src will be set to that value in dst.
+	// Elements of repeated fields will be appended.
+	//
+	// Merge may panic if called with a different argument type than the receiver.
+	Merge(src Message)
+}
+
+// generatedMerger is the custom merge method that generated protos will have.
+// We must add this method since a generate Merge method will conflict with
+// many existing protos that have a Merge data field already defined.
+type generatedMerger interface {
+	XXX_Merge(src Message)
 }
 
 // Merge merges src into dst.
@@ -58,17 +75,24 @@ func Clone(pb Message) Message {
 // Elements of repeated fields will be appended.
 // Merge panics if src and dst are not the same type, or if dst is nil.
 func Merge(dst, src Message) {
+	if m, ok := dst.(Merger); ok {
+		m.Merge(src)
+		return
+	}
+
 	in := reflect.ValueOf(src)
 	out := reflect.ValueOf(dst)
 	if out.IsNil() {
 		panic("proto: nil destination")
 	}
 	if in.Type() != out.Type() {
-		// Explicit test prior to mergeStruct so that mistyped nils will fail
-		panic("proto: type mismatch")
+		panic(fmt.Sprintf("proto.Merge(%T, %T) type mismatch", dst, src))
 	}
 	if in.IsNil() {
-		// Merging nil into non-nil is a quiet no-op
+		return // Merge from nil src is a noop
+	}
+	if m, ok := dst.(generatedMerger); ok {
+		m.XXX_Merge(src)
 		return
 	}
 	mergeStruct(out.Elem(), in.Elem())
@@ -84,9 +108,15 @@ func mergeStruct(out, in reflect.Value) {
 		mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i])
 	}
 
-	if emIn, ok := in.Addr().Interface().(extendableProto); ok {
-		emOut := out.Addr().Interface().(extendableProto)
-		mergeExtension(emOut.ExtensionMap(), emIn.ExtensionMap())
+	if emIn, err := extendable(in.Addr().Interface()); err == nil {
+		emOut, _ := extendable(out.Addr().Interface())
+		mIn, muIn := emIn.extensionsRead()
+		if mIn != nil {
+			mOut := emOut.extensionsWrite()
+			muIn.Lock()
+			mergeExtension(mOut, mIn)
+			muIn.Unlock()
+		}
 	}
 
 	uf := in.FieldByName("XXX_unrecognized")

vendor/github.com/golang/protobuf/proto/equal.go

@@ -54,13 +54,17 @@ Equality is defined in this way:
     in a proto3 .proto file, fields are not "set"; specifically,
     zero length proto3 "bytes" fields are equal (nil == {}).
   - Two repeated fields are equal iff their lengths are the same,
-    and their corresponding elements are equal (a "bytes" field,
-    although represented by []byte, is not a repeated field)
+    and their corresponding elements are equal. Note a "bytes" field,
+    although represented by []byte, is not a repeated field and the
+    rule for the scalar fields described above applies.
   - Two unset fields are equal.
   - Two unknown field sets are equal if their current
     encoded state is equal.
   - Two extension sets are equal iff they have corresponding
     elements that are pairwise equal.
+  - Two map fields are equal iff their lengths are the same,
+    and they contain the same set of elements. Zero-length map
+    fields are equal.
   - Every other combination of things are not equal.
 
 The return value is undefined if a and b are not protocol buffers.
@@ -105,15 +109,6 @@ func equalStruct(v1, v2 reflect.Value) bool {
 				// set/unset mismatch
 				return false
 			}
-			b1, ok := f1.Interface().(raw)
-			if ok {
-				b2 := f2.Interface().(raw)
-				// RawMessage
-				if !bytes.Equal(b1.Bytes(), b2.Bytes()) {
-					return false
-				}
-				continue
-			}
 			f1, f2 = f1.Elem(), f2.Elem()
 		}
 		if !equalAny(f1, f2, sprop.Prop[i]) {
@@ -121,9 +116,16 @@ func equalStruct(v1, v2 reflect.Value) bool {
 		}
 	}
 
+	if em1 := v1.FieldByName("XXX_InternalExtensions"); em1.IsValid() {
+		em2 := v2.FieldByName("XXX_InternalExtensions")
+		if !equalExtensions(v1.Type(), em1.Interface().(XXX_InternalExtensions), em2.Interface().(XXX_InternalExtensions)) {
+			return false
+		}
+	}
+
 	if em1 := v1.FieldByName("XXX_extensions"); em1.IsValid() {
 		em2 := v2.FieldByName("XXX_extensions")
-		if !equalExtensions(v1.Type(), em1.Interface().(map[int32]Extension), em2.Interface().(map[int32]Extension)) {
+		if !equalExtMap(v1.Type(), em1.Interface().(map[int32]Extension), em2.Interface().(map[int32]Extension)) {
 			return false
 		}
 	}
@@ -135,11 +137,7 @@ func equalStruct(v1, v2 reflect.Value) bool {
 
 	u1 := uf.Bytes()
 	u2 := v2.FieldByName("XXX_unrecognized").Bytes()
-	if !bytes.Equal(u1, u2) {
-		return false
-	}
-
-	return true
+	return bytes.Equal(u1, u2)
 }
 
 // v1 and v2 are known to have the same type.
@@ -184,6 +182,13 @@ func equalAny(v1, v2 reflect.Value, prop *Properties) bool {
 		}
 		return true
 	case reflect.Ptr:
+		// Maps may have nil values in them, so check for nil.
+		if v1.IsNil() && v2.IsNil() {
+			return true
+		}
+		if v1.IsNil() != v2.IsNil() {
+			return false
+		}
 		return equalAny(v1.Elem(), v2.Elem(), prop)
 	case reflect.Slice:
 		if v1.Type().Elem().Kind() == reflect.Uint8 {
@@ -223,8 +228,14 @@ func equalAny(v1, v2 reflect.Value, prop *Properties) bool {
 }
 
 // base is the struct type that the extensions are based on.
-// em1 and em2 are extension maps.
-func equalExtensions(base reflect.Type, em1, em2 map[int32]Extension) bool {
+// x1 and x2 are InternalExtensions.
+func equalExtensions(base reflect.Type, x1, x2 XXX_InternalExtensions) bool {
+	em1, _ := x1.extensionsRead()
+	em2, _ := x2.extensionsRead()
+	return equalExtMap(base, em1, em2)
+}
+
+func equalExtMap(base reflect.Type, em1, em2 map[int32]Extension) bool {
 	if len(em1) != len(em2) {
 		return false
 	}
@@ -237,6 +248,15 @@ func equalExtensions(base reflect.Type, em1, em2 map[int32]Extension) bool {
 
 		m1, m2 := e1.value, e2.value
 
+		if m1 == nil && m2 == nil {
+			// Both have only encoded form.
+			if bytes.Equal(e1.enc, e2.enc) {
+				continue
+			}
+			// The bytes are different, but the extensions might still be
+			// equal. We need to decode them to compare.
+		}
+
 		if m1 != nil && m2 != nil {
 			// Both are unencoded.
 			if !equalAny(reflect.ValueOf(m1), reflect.ValueOf(m2), nil) {
@@ -252,8 +272,12 @@ func equalExtensions(base reflect.Type, em1, em2 map[int32]Extension) bool {
 			desc = m[extNum]
 		}
 		if desc == nil {
+			// If both have only encoded form and the bytes are the same,
+			// it is handled above. We get here when the bytes are different.
+			// We don't know how to decode it, so just compare them as byte
+			// slices.
 			log.Printf("proto: don't know how to compare extension %d of %v", extNum, base)
-			continue
+			return false
 		}
 		var err error
 		if m1 == nil {

vendor/github.com/golang/protobuf/proto/lib.go

@@ -73,7 +73,6 @@ for a protocol buffer variable v:
 When the .proto file specifies `syntax="proto3"`, there are some differences:
 
   - Non-repeated fields of non-message type are values instead of pointers.
-  - Getters are only generated for message and oneof fields.
   - Enum types do not get an Enum method.
 
 The simplest way to describe this is to see an example.
@@ -274,6 +273,67 @@ import (
 	"sync"
 )
 
+// RequiredNotSetError is an error type returned by either Marshal or Unmarshal.
+// Marshal reports this when a required field is not initialized.
+// Unmarshal reports this when a required field is missing from the wire data.
+type RequiredNotSetError struct{ field string }
+
+func (e *RequiredNotSetError) Error() string {
+	if e.field == "" {
+		return fmt.Sprintf("proto: required field not set")
+	}
+	return fmt.Sprintf("proto: required field %q not set", e.field)
+}
+func (e *RequiredNotSetError) RequiredNotSet() bool {
+	return true
+}
+
+type invalidUTF8Error struct{ field string }
+
+func (e *invalidUTF8Error) Error() string {
+	if e.field == "" {
+		return "proto: invalid UTF-8 detected"
+	}
+	return fmt.Sprintf("proto: field %q contains invalid UTF-8", e.field)
+}
+func (e *invalidUTF8Error) InvalidUTF8() bool {
+	return true
+}
+
+// errInvalidUTF8 is a sentinel error to identify fields with invalid UTF-8.
+// This error should not be exposed to the external API as such errors should
+// be recreated with the field information.
+var errInvalidUTF8 = &invalidUTF8Error{}
+
+// isNonFatal reports whether the error is either a RequiredNotSet error
+// or a InvalidUTF8 error.
+func isNonFatal(err error) bool {
+	if re, ok := err.(interface{ RequiredNotSet() bool }); ok && re.RequiredNotSet() {
+		return true
+	}
+	if re, ok := err.(interface{ InvalidUTF8() bool }); ok && re.InvalidUTF8() {
+		return true
+	}
+	return false
+}
+
+type nonFatal struct{ E error }
+
+// Merge merges err into nf and reports whether it was successful.
+// Otherwise it returns false for any fatal non-nil errors.
+func (nf *nonFatal) Merge(err error) (ok bool) {
+	if err == nil {
+		return true // not an error
+	}
+	if !isNonFatal(err) {
+		return false // fatal error
+	}
+	if nf.E == nil {
+		nf.E = err // store first instance of non-fatal error
+	}
+	return true
+}
+
 // Message is implemented by generated protocol buffer messages.
 type Message interface {
 	Reset()
@@ -308,18 +368,9 @@ func GetStats() Stats { return stats }
 // temporary Buffer and are fine for most applications.
 type Buffer struct {
 	buf   []byte // encode/decode byte stream
-	index int    // write point
+	index int    // read point
 
-	// pools of basic types to amortize allocation.
-	bools   []bool
-	uint32s []uint32
-	uint64s []uint64
-
-	// extra pools, only used with pointer_reflect.go
-	int32s   []int32
-	int64s   []int64
-	float32s []float32
-	float64s []float64
+	deterministic bool
 }
 
 // NewBuffer allocates a new Buffer and initializes its internal data to
@@ -344,6 +395,30 @@ func (p *Buffer) SetBuf(s []byte) {
 // Bytes returns the contents of the Buffer.
 func (p *Buffer) Bytes() []byte { return p.buf }
 
+// SetDeterministic sets whether to use deterministic serialization.
+//
+// Deterministic serialization guarantees that for a given binary, equal
+// messages will always be serialized to the same bytes. This implies:
+//
+//   - Repeated serialization of a message will return the same bytes.
+//   - Different processes of the same binary (which may be executing on
+//     different machines) will serialize equal messages to the same bytes.
+//
+// Note that the deterministic serialization is NOT canonical across
+// languages. It is not guaranteed to remain stable over time. It is unstable
+// across different builds with schema changes due to unknown fields.
+// Users who need canonical serialization (e.g., persistent storage in a
+// canonical form, fingerprinting, etc.) should define their own
+// canonicalization specification and implement their own serializer rather
+// than relying on this API.
+//
+// If deterministic serialization is requested, map entries will be sorted
+// by keys in lexographical order. This is an implementation detail and
+// subject to change.
+func (p *Buffer) SetDeterministic(deterministic bool) {
+	p.deterministic = deterministic
+}
+
 /*
  * Helper routines for simplifying the creation of optional fields of basic type.
  */
@@ -832,22 +907,12 @@ func fieldDefault(ft reflect.Type, prop *Properties) (sf *scalarField, nestedMes
 	return sf, false, nil
 }
 
+// mapKeys returns a sort.Interface to be used for sorting the map keys.
 // Map fields may have key types of non-float scalars, strings and enums.
-// The easiest way to sort them in some deterministic order is to use fmt.
-// If this turns out to be inefficient we can always consider other options,
-// such as doing a Schwartzian transform.
-
 func mapKeys(vs []reflect.Value) sort.Interface {
-	s := mapKeySorter{
-		vs: vs,
-		// default Less function: textual comparison
-		less: func(a, b reflect.Value) bool {
-			return fmt.Sprint(a.Interface()) < fmt.Sprint(b.Interface())
-		},
-	}
+	s := mapKeySorter{vs: vs}
 
-	// Type specialization per https://developers.google.com/protocol-buffers/docs/proto#maps;
-	// numeric keys are sorted numerically.
+	// Type specialization per https://developers.google.com/protocol-buffers/docs/proto#maps.
 	if len(vs) == 0 {
 		return s
 	}
@@ -856,6 +921,12 @@ func mapKeys(vs []reflect.Value) sort.Interface {
 		s.less = func(a, b reflect.Value) bool { return a.Int() < b.Int() }
 	case reflect.Uint32, reflect.Uint64:
 		s.less = func(a, b reflect.Value) bool { return a.Uint() < b.Uint() }
+	case reflect.Bool:
+		s.less = func(a, b reflect.Value) bool { return !a.Bool() && b.Bool() } // false < true
+	case reflect.String:
+		s.less = func(a, b reflect.Value) bool { return a.String() < b.String() }
+	default:
+		panic(fmt.Sprintf("unsupported map key type: %v", vs[0].Kind()))
 	}
 
 	return s
@@ -889,6 +960,20 @@ func isProto3Zero(v reflect.Value) bool {
 	return false
 }
 
+// ProtoPackageIsVersion2 is referenced from generated protocol buffer files
+// to assert that that code is compatible with this version of the proto package.
+const ProtoPackageIsVersion2 = true
+
 // ProtoPackageIsVersion1 is referenced from generated protocol buffer files
 // to assert that that code is compatible with this version of the proto package.
 const ProtoPackageIsVersion1 = true
+
+// InternalMessageInfo is a type used internally by generated .pb.go files.
+// This type is not intended to be used by non-generated code.
+// This type is not subject to any compatibility guarantee.
+type InternalMessageInfo struct {
+	marshal   *marshalInfo
+	unmarshal *unmarshalInfo
+	merge     *mergeInfo
+	discard   *discardInfo
+}

vendor/github.com/golang/protobuf/proto/message_set.go

@@ -42,6 +42,7 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
+	"sync"
 )
 
 // errNoMessageTypeID occurs when a protocol buffer does not have a message type ID.
@@ -94,10 +95,7 @@ func (ms *messageSet) find(pb Message) *_MessageSet_Item {
 }
 
 func (ms *messageSet) Has(pb Message) bool {
-	if ms.find(pb) != nil {
-		return true
-	}
-	return false
+	return ms.find(pb) != nil
 }
 
 func (ms *messageSet) Unmarshal(pb Message) error {
@@ -149,36 +147,54 @@ func skipVarint(buf []byte) []byte {
 
 // MarshalMessageSet encodes the extension map represented by m in the message set wire format.
 // It is called by generated Marshal methods on protocol buffer messages with the message_set_wire_format option.
-func MarshalMessageSet(m map[int32]Extension) ([]byte, error) {
-	if err := encodeExtensionMap(m); err != nil {
-		return nil, err
-	}
+func MarshalMessageSet(exts interface{}) ([]byte, error) {
+	return marshalMessageSet(exts, false)
+}
 
-	// Sort extension IDs to provide a deterministic encoding.
-	// See also enc_map in encode.go.
-	ids := make([]int, 0, len(m))
-	for id := range m {
-		ids = append(ids, int(id))
-	}
-	sort.Ints(ids)
+// marshaMessageSet implements above function, with the opt to turn on / off deterministic during Marshal.
+func marshalMessageSet(exts interface{}, deterministic bool) ([]byte, error) {
+	switch exts := exts.(type) {
+	case *XXX_InternalExtensions:
+		var u marshalInfo
+		siz := u.sizeMessageSet(exts)
+		b := make([]byte, 0, siz)
+		return u.appendMessageSet(b, exts, deterministic)
 
-	ms := &messageSet{Item: make([]*_MessageSet_Item, 0, len(m))}
-	for _, id := range ids {
-		e := m[int32(id)]
-		// Remove the wire type and field number varint, as well as the length varint.
-		msg := skipVarint(skipVarint(e.enc))
+	case map[int32]Extension:
+		// This is an old-style extension map.
+		// Wrap it in a new-style XXX_InternalExtensions.
+		ie := XXX_InternalExtensions{
+			p: &struct {
+				mu           sync.Mutex
+				extensionMap map[int32]Extension
+			}{
+				extensionMap: exts,
+			},
+		}
 
-		ms.Item = append(ms.Item, &_MessageSet_Item{
-			TypeId:  Int32(int32(id)),
-			Message: msg,
-		})
+		var u marshalInfo
+		siz := u.sizeMessageSet(&ie)
+		b := make([]byte, 0, siz)
+		return u.appendMessageSet(b, &ie, deterministic)
+
+	default:
+		return nil, errors.New("proto: not an extension map")
 	}
-	return Marshal(ms)
 }
 
 // UnmarshalMessageSet decodes the extension map encoded in buf in the message set wire format.
-// It is called by generated Unmarshal methods on protocol buffer messages with the message_set_wire_format option.
-func UnmarshalMessageSet(buf []byte, m map[int32]Extension) error {
+// It is called by Unmarshal methods on protocol buffer messages with the message_set_wire_format option.
+func UnmarshalMessageSet(buf []byte, exts interface{}) error {
+	var m map[int32]Extension
+	switch exts := exts.(type) {
+	case *XXX_InternalExtensions:
+		m = exts.extensionsWrite()
+	case map[int32]Extension:
+		m = exts
+	default:
+		return errors.New("proto: not an extension map")
+	}
+
 	ms := new(messageSet)
 	if err := Unmarshal(buf, ms); err != nil {
 		return err
@@ -209,7 +225,24 @@ func UnmarshalMessageSet(buf []byte, m map[int32]Extension) error {
 
 // MarshalMessageSetJSON encodes the extension map represented by m in JSON format.
 // It is called by generated MarshalJSON methods on protocol buffer messages with the message_set_wire_format option.
-func MarshalMessageSetJSON(m map[int32]Extension) ([]byte, error) {
+func MarshalMessageSetJSON(exts interface{}) ([]byte, error) {
+	var m map[int32]Extension
+	switch exts := exts.(type) {
+	case *XXX_InternalExtensions:
+		var mu sync.Locker
+		m, mu = exts.extensionsRead()
+		if m != nil {
+			// Keep the extensions map locked until we're done marshaling to prevent
+			// races between marshaling and unmarshaling the lazily-{en,de}coded
+			// values.
+			mu.Lock()
+			defer mu.Unlock()
+		}
+	case map[int32]Extension:
+		m = exts
+	default:
+		return nil, errors.New("proto: not an extension map")
+	}
 	var b bytes.Buffer
 	b.WriteByte('{')
 
@@ -222,15 +255,16 @@ func MarshalMessageSetJSON(m map[int32]Extension) ([]byte, error) {
 
 	for i, id := range ids {
 		ext := m[id]
-		if i > 0 {
-			b.WriteByte(',')
-		}
-
 		msd, ok := messageSetMap[id]
 		if !ok {
 			// Unknown type; we can't render it, so skip it.
 			continue
 		}
+
+		if i > 0 && b.Len() > 1 {
+			b.WriteByte(',')
+		}
+
 		fmt.Fprintf(&b, `"[%s]":`, msd.name)
 
 		x := ext.value
@@ -252,7 +286,7 @@ func MarshalMessageSetJSON(m map[int32]Extension) ([]byte, error) {
 
 // UnmarshalMessageSetJSON decodes the extension map encoded in buf in JSON format.
 // It is called by generated UnmarshalJSON methods on protocol buffer messages with the message_set_wire_format option.
-func UnmarshalMessageSetJSON(buf []byte, m map[int32]Extension) error {
+func UnmarshalMessageSetJSON(buf []byte, exts interface{}) error {
 	// Common-case fast path.
 	if len(buf) == 0 || bytes.Equal(buf, []byte("{}")) {
 		return nil

vendor/github.com/golang/protobuf/proto/text.go

@@ -50,7 +50,6 @@ import (
 var (
 	newline         = []byte("\n")
 	spaces          = []byte("                                        ")
-	gtNewline       = []byte(">\n")
 	endBraceNewline = []byte("}\n")
 	backslashN      = []byte{'\\', 'n'}
 	backslashR      = []byte{'\\', 'r'}
@@ -154,7 +153,7 @@ func (w *textWriter) indent() { w.ind++ }
 
 func (w *textWriter) unindent() {
 	if w.ind == 0 {
-		log.Printf("proto: textWriter unindented too far")
+		log.Print("proto: textWriter unindented too far")
 		return
 	}
 	w.ind--
@@ -170,11 +169,6 @@ func writeName(w *textWriter, props *Properties) error {
 	return nil
 }
 
-// raw is the interface satisfied by RawMessage.
-type raw interface {
-	Bytes() []byte
-}
-
 func requiresQuotes(u string) bool {
 	// When type URL contains any characters except [0-9A-Za-z./\-]*, it must be quoted.
 	for _, ch := range u {
@@ -269,6 +263,10 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error {
 		props := sprops.Prop[i]
 		name := st.Field(i).Name
 
+		if name == "XXX_NoUnkeyedLiteral" {
+			continue
+		}
+
 		if strings.HasPrefix(name, "XXX_") {
 			// There are two XXX_ fields:
 			//   XXX_unrecognized []byte
@@ -355,7 +353,7 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error {
 						return err
 					}
 				}
-				if err := tm.writeAny(w, key, props.mkeyprop); err != nil {
+				if err := tm.writeAny(w, key, props.MapKeyProp); err != nil {
 					return err
 				}
 				if err := w.WriteByte('\n'); err != nil {
@@ -372,7 +370,7 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error {
 							return err
 						}
 					}
-					if err := tm.writeAny(w, val, props.mvalprop); err != nil {
+					if err := tm.writeAny(w, val, props.MapValProp); err != nil {
 						return err
 					}
 					if err := w.WriteByte('\n'); err != nil {
@@ -436,12 +434,6 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error {
 				return err
 			}
 		}
-		if b, ok := fv.Interface().(raw); ok {
-			if err := writeRaw(w, b.Bytes()); err != nil {
-				return err
-			}
-			continue
-		}
 
 		// Enums have a String method, so writeAny will work fine.
 		if err := tm.writeAny(w, fv, props); err != nil {
@@ -455,7 +447,7 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error {
 
 	// Extensions (the XXX_extensions field).
 	pv := sv.Addr()
-	if pv.Type().Implements(extendableProtoType) {
+	if _, err := extendable(pv.Interface()); err == nil {
 		if err := tm.writeExtensions(w, pv); err != nil {
 			return err
 		}
@@ -464,27 +456,6 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error {
 	return nil
 }
 
-// writeRaw writes an uninterpreted raw message.
-func writeRaw(w *textWriter, b []byte) error {
-	if err := w.WriteByte('<'); err != nil {
-		return err
-	}
-	if !w.compact {
-		if err := w.WriteByte('\n'); err != nil {
-			return err
-		}
-	}
-	w.indent()
-	if err := writeUnknownStruct(w, b); err != nil {
-		return err
-	}
-	w.unindent()
-	if err := w.WriteByte('>'); err != nil {
-		return err
-	}
-	return nil
-}
-
 // writeAny writes an arbitrary field.
 func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Properties) error {
 	v = reflect.Indirect(v)
@@ -513,7 +484,7 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert
 	switch v.Kind() {
 	case reflect.Slice:
 		// Should only be a []byte; repeated fields are handled in writeStruct.
-		if err := writeString(w, string(v.Interface().([]byte))); err != nil {
+		if err := writeString(w, string(v.Bytes())); err != nil {
 			return err
 		}
 	case reflect.String:
@@ -535,6 +506,19 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert
 			}
 		}
 		w.indent()
+		if v.CanAddr() {
+			// Calling v.Interface on a struct causes the reflect package to
+			// copy the entire struct. This is racy with the new Marshaler
+			// since we atomically update the XXX_sizecache.
+			//
+			// Thus, we retrieve a pointer to the struct if possible to avoid
+			// a race since v.Interface on the pointer doesn't copy the struct.
+			//
+			// If v is not addressable, then we are not worried about a race
+			// since it implies that the binary Marshaler cannot possibly be
+			// mutating this value.
+			v = v.Addr()
+		}
 		if etm, ok := v.Interface().(encoding.TextMarshaler); ok {
 			text, err := etm.MarshalText()
 			if err != nil {
@@ -543,8 +527,13 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert
 			if _, err = w.Write(text); err != nil {
 				return err
 			}
-		} else if err := tm.writeStruct(w, v); err != nil {
-			return err
+		} else {
+			if v.Kind() == reflect.Ptr {
+				v = v.Elem()
+			}
+			if err := tm.writeStruct(w, v); err != nil {
+				return err
+			}
 		}
 		w.unindent()
 		if err := w.WriteByte(ket); err != nil {
@@ -689,17 +678,22 @@ func (s int32Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 // pv is assumed to be a pointer to a protocol message struct that is extendable.
 func (tm *TextMarshaler) writeExtensions(w *textWriter, pv reflect.Value) error {
 	emap := extensionMaps[pv.Type().Elem()]
-	ep := pv.Interface().(extendableProto)
+	ep, _ := extendable(pv.Interface())
 
 	// Order the extensions by ID.
 	// This isn't strictly necessary, but it will give us
 	// canonical output, which will also make testing easier.
-	m := ep.ExtensionMap()
+	m, mu := ep.extensionsRead()
+	if m == nil {
+		return nil
+	}
+	mu.Lock()
 	ids := make([]int32, 0, len(m))
 	for id := range m {
 		ids = append(ids, id)
 	}
 	sort.Sort(int32Slice(ids))
+	mu.Unlock()
 
 	for _, extNum := range ids {
 		ext := m[extNum]

vendor/github.com/golang/protobuf/proto/text_parser.go

@@ -44,6 +44,9 @@ import (
 	"unicode/utf8"
 )
 
+// Error string emitted when deserializing Any and fields are already set
+const anyRepeatedlyUnpacked = "Any message unpacked multiple times, or %q already set"
+
 type ParseError struct {
 	Message string
 	Line    int // 1-based line number
@@ -203,7 +206,6 @@ func (p *textParser) advance() {
 
 var (
 	errBadUTF8 = errors.New("proto: bad UTF-8")
-	errBadHex  = errors.New("proto: bad hexadecimal")
 )
 
 func unquoteC(s string, quote rune) (string, error) {
@@ -274,60 +276,47 @@ func unescape(s string) (ch string, tail string, err error) {
 		return "?", s, nil // trigraph workaround
 	case '\'', '"', '\\':
 		return string(r), s, nil
-	case '0', '1', '2', '3', '4', '5', '6', '7', 'x', 'X':
+	case '0', '1', '2', '3', '4', '5', '6', '7':
 		if len(s) < 2 {
 			return "", "", fmt.Errorf(`\%c requires 2 following digits`, r)
 		}
-		base := 8
-		ss := s[:2]
+		ss := string(r) + s[:2]
 		s = s[2:]
-		if r == 'x' || r == 'X' {
-			base = 16
-		} else {
-			ss = string(r) + ss
-		}
-		i, err := strconv.ParseUint(ss, base, 8)
+		i, err := strconv.ParseUint(ss, 8, 8)
 		if err != nil {
-			return "", "", err
+			return "", "", fmt.Errorf(`\%s contains non-octal digits`, ss)
 		}
 		return string([]byte{byte(i)}), s, nil
-	case 'u', 'U':
-		n := 4
-		if r == 'U' {
+	case 'x', 'X', 'u', 'U':
+		var n int
+		switch r {
+		case 'x', 'X':
+			n = 2
+		case 'u':
+			n = 4
+		case 'U':
 			n = 8
 		}
 		if len(s) < n {
-			return "", "", fmt.Errorf(`\%c requires %d digits`, r, n)
-		}
-
-		bs := make([]byte, n/2)
-		for i := 0; i < n; i += 2 {
-			a, ok1 := unhex(s[i])
-			b, ok2 := unhex(s[i+1])
-			if !ok1 || !ok2 {
-				return "", "", errBadHex
-			}
-			bs[i/2] = a<<4 | b
+			return "", "", fmt.Errorf(`\%c requires %d following digits`, r, n)
 		}
+		ss := s[:n]
 		s = s[n:]
-		return string(bs), s, nil
+		i, err := strconv.ParseUint(ss, 16, 64)
+		if err != nil {
+			return "", "", fmt.Errorf(`\%c%s contains non-hexadecimal digits`, r, ss)
+		}
+		if r == 'x' || r == 'X' {
+			return string([]byte{byte(i)}), s, nil
+		}
+		if i > utf8.MaxRune {
+			return "", "", fmt.Errorf(`\%c%s is not a valid Unicode code point`, r, ss)
+		}
+		return string(i), s, nil
 	}
 	return "", "", fmt.Errorf(`unknown escape \%c`, r)
 }
 
-// Adapted from src/pkg/strconv/quote.go.
-func unhex(b byte) (v byte, ok bool) {
-	switch {
-	case '0' <= b && b <= '9':
-		return b - '0', true
-	case 'a' <= b && b <= 'f':
-		return b - 'a' + 10, true
-	case 'A' <= b && b <= 'F':
-		return b - 'A' + 10, true
-	}
-	return 0, false
-}
-
 // Back off the parser by one token. Can only be done between calls to next().
 // It makes the next advance() a no-op.
 func (p *textParser) back() { p.backed = true }
@@ -508,8 +497,16 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error {
 				if err != nil {
 					return p.errorf("failed to marshal message of type %q: %v", messageName, err)
 				}
+				if fieldSet["type_url"] {
+					return p.errorf(anyRepeatedlyUnpacked, "type_url")
+				}
+				if fieldSet["value"] {
+					return p.errorf(anyRepeatedlyUnpacked, "value")
+				}
 				sv.FieldByName("TypeUrl").SetString(extName)
 				sv.FieldByName("Value").SetBytes(b)
+				fieldSet["type_url"] = true
+				fieldSet["value"] = true
 				continue
 			}
 
@@ -550,7 +547,7 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error {
 				}
 				reqFieldErr = err
 			}
-			ep := sv.Addr().Interface().(extendableProto)
+			ep := sv.Addr().Interface().(Message)
 			if !rep {
 				SetExtension(ep, desc, ext.Interface())
 			} else {
@@ -581,7 +578,11 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error {
 			props = oop.Prop
 			nv := reflect.New(oop.Type.Elem())
 			dst = nv.Elem().Field(0)
-			sv.Field(oop.Field).Set(nv)
+			field := sv.Field(oop.Field)
+			if !field.IsNil() {
+				return p.errorf("field '%s' would overwrite already parsed oneof '%s'", name, sv.Type().Field(oop.Field).Name)
+			}
+			field.Set(nv)
 		}
 		if !dst.IsValid() {
 			return p.errorf("unknown field name %q in %v", name, st)
@@ -602,8 +603,9 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error {
 
 			// The map entry should be this sequence of tokens:
 			//	< key : KEY value : VALUE >
-			// Technically the "key" and "value" could come in any order,
-			// but in practice they won't.
+			// However, implementations may omit key or value, and technically
+			// we should support them in any order.  See b/28924776 for a time
+			// this went wrong.
 
 			tok := p.next()
 			var terminator string
@@ -615,32 +617,39 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error {
 			default:
 				return p.errorf("expected '{' or '<', found %q", tok.value)
 			}
-			if err := p.consumeToken("key"); err != nil {
-				return err
-			}
-			if err := p.consumeToken(":"); err != nil {
-				return err
-			}
-			if err := p.readAny(key, props.mkeyprop); err != nil {
-				return err
-			}
-			if err := p.consumeOptionalSeparator(); err != nil {
-				return err
-			}
-			if err := p.consumeToken("value"); err != nil {
-				return err
-			}
-			if err := p.checkForColon(props.mvalprop, dst.Type().Elem()); err != nil {
-				return err
-			}
-			if err := p.readAny(val, props.mvalprop); err != nil {
-				return err
-			}
-			if err := p.consumeOptionalSeparator(); err != nil {
-				return err
-			}
-			if err := p.consumeToken(terminator); err != nil {
-				return err
+			for {
+				tok := p.next()
+				if tok.err != nil {
+					return tok.err
+				}
+				if tok.value == terminator {
+					break
+				}
+				switch tok.value {
+				case "key":
+					if err := p.consumeToken(":"); err != nil {
+						return err
+					}
+					if err := p.readAny(key, props.MapKeyProp); err != nil {
+						return err
+					}
+					if err := p.consumeOptionalSeparator(); err != nil {
+						return err
+					}
+				case "value":
+					if err := p.checkForColon(props.MapValProp, dst.Type().Elem()); err != nil {
+						return err
+					}
+					if err := p.readAny(val, props.MapValProp); err != nil {
+						return err
+					}
+					if err := p.consumeOptionalSeparator(); err != nil {
+						return err
+					}
+				default:
+					p.back()
+					return p.errorf(`expected "key", "value", or %q, found %q`, terminator, tok.value)
+				}
 			}
 
 			dst.SetMapIndex(key, val)
@@ -663,7 +672,8 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error {
 				return err
 			}
 			reqFieldErr = err
-		} else if props.Required {
+		}
+		if props.Required {
 			reqCount--
 		}
 
@@ -704,6 +714,9 @@ func (p *textParser) consumeExtName() (string, error) {
 		if tok.err != nil {
 			return "", p.errorf("unrecognized type_url or extension name: %s", tok.err)
 		}
+		if p.done && tok.value != "]" {
+			return "", p.errorf("unclosed type_url or extension name")
+		}
 	}
 	return strings.Join(parts, ""), nil
 }
@@ -772,12 +785,12 @@ func (p *textParser) readAny(v reflect.Value, props *Properties) error {
 		fv.Set(reflect.Append(fv, reflect.New(at.Elem()).Elem()))
 		return p.readAny(fv.Index(fv.Len()-1), props)
 	case reflect.Bool:
-		// Either "true", "false", 1 or 0.
+		// true/1/t/True or false/f/0/False.
 		switch tok.value {
-		case "true", "1":
+		case "true", "1", "t", "True":
 			fv.SetBool(true)
 			return nil
-		case "false", "0":
+		case "false", "0", "f", "False":
 			fv.SetBool(false)
 			return nil
 		}
@@ -859,13 +872,9 @@ func (p *textParser) readAny(v reflect.Value, props *Properties) error {
 // UnmarshalText returns *RequiredNotSetError.
 func UnmarshalText(s string, pb Message) error {
 	if um, ok := pb.(encoding.TextUnmarshaler); ok {
-		err := um.UnmarshalText([]byte(s))
-		return err
+		return um.UnmarshalText([]byte(s))
 	}
 	pb.Reset()
 	v := reflect.ValueOf(pb)
-	if pe := newTextParser(s).readStruct(v.Elem(), ""); pe != nil {
-		return pe
-	}
-	return nil
+	return newTextParser(s).readStruct(v.Elem(), "")
 }

vendor/github.com/matttproud/golang_protobuf_extensions/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/matttproud/golang_protobuf_extensions/NOTICE

@@ -0,0 +1 @@
+Copyright 2012 Matt T. Proud (matt.proud@gmail.com)