LFS Extensions: pointer file manipulation

commands/command_pointer.go

@@ -52,7 +52,7 @@ func pointerCommand(cmd *cobra.Command, args []string) {
 			os.Exit(1)
 		}
 
-		ptr := lfs.NewPointer(hex.EncodeToString(oidHash.Sum(nil)), size)
+		ptr := lfs.NewPointer(hex.EncodeToString(oidHash.Sum(nil)), size, nil)
 		fmt.Printf("Git LFS pointer for %s\n\n", pointerFile)
 		buf := &bytes.Buffer{}
 		lfs.EncodePointer(io.MultiWriter(os.Stdout, buf), ptr)

lfs/pointer.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"os"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 )
@@ -18,29 +19,43 @@ var (
 		"https://hawser.github.com/spec/v1",  // pre-release
 		"https://git-lfs.github.com/spec/v1", // public launch
 	}
-	latest = "https://git-lfs.github.com/spec/v1"
+	latest      = "https://git-lfs.github.com/spec/v1"
-
+	oidType     = "sha256"
-	oidType  = "sha256"
+	oidRE       = regexp.MustCompile(`\A[[:alnum:]]{64}`)
-	oidRE    = regexp.MustCompile(`\A[0-9a-fA-F]{64}`)
-	template = `version %s
-oid sha256:%s
-size %d
-`
 	matcherRE   = regexp.MustCompile("git-media|hawser|git-lfs")
+	extRE       = regexp.MustCompile(`\Aext-\d{1}-\w+`)
 	pointerKeys = []string{"version", "oid", "size"}
 
 	NotAPointerError = errors.New("Not a valid Git LFS pointer file.")
 )
 
 type Pointer struct {
-	Version string
+	Version    string
-	Oid     string
+	Oid        string
-	Size    int64
+	Size       int64
-	OidType string
+	OidType    string
+	Extensions []PointerExtension
 }
 
-func NewPointer(oid string, size int64) *Pointer {
+type PointerExtension struct {
-	return &Pointer{latest, oid, size, oidType}
+	Name     string
+	Priority int
+	Oid      string
+	OidType  string
+}
+
+type ByPriority []PointerExtension
+
+func (p ByPriority) Len() int           { return len(p) }
+func (p ByPriority) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p ByPriority) Less(i, j int) bool { return p[i].Priority < p[j].Priority }
+
+func NewPointer(oid string, size int64, exts []PointerExtension) *Pointer {
+	return &Pointer{latest, oid, size, oidType, exts}
+}
+
+func NewPointerExtension(name string, priority int, oid string) *PointerExtension {
+	return &PointerExtension{name, priority, oid, oidType}
 }
 
 func (p *Pointer) Smudge(writer io.Writer, workingfile string, cb CopyCallback) error {
@@ -52,7 +67,14 @@ func (p *Pointer) Encode(writer io.Writer) (int, error) {
 }
 
 func (p *Pointer) Encoded() string {
-	return fmt.Sprintf(template, latest, p.Oid, p.Size)
+	var buffer bytes.Buffer
+	buffer.WriteString(fmt.Sprintf("version %s\n", latest))
+	for _, ext := range p.Extensions {
+		buffer.WriteString(fmt.Sprintf("ext-%d-%s %s:%s\n", ext.Priority, ext.Name, ext.OidType, ext.Oid))
+	}
+	buffer.WriteString(fmt.Sprintf("oid %s:%s\n", p.OidType, p.Oid))
+	buffer.WriteString(fmt.Sprintf("size %d\n", p.Size))
+	return buffer.String()
 }
 
 func EncodePointer(writer io.Writer, pointer *Pointer) (int, error) {
@@ -108,48 +130,106 @@ func verifyVersion(version string) error {
 }
 
 func decodeKV(data []byte) (*Pointer, error) {
-	parsed, err := decodeKVData(data)
+	kvps, exts, err := decodeKVData(data)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := verifyVersion(parsed["version"]); err != nil {
+	if err := verifyVersion(kvps["version"]); err != nil {
 		return nil, err
 	}
 
-	oidValue, ok := parsed["oid"]
+	value, ok := kvps["oid"]
 	if !ok {
 		return nil, errors.New("Invalid Oid")
 	}
 
-	oidParts := strings.SplitN(oidValue, ":", 2)
+	oid, err := parseOid(value)
-	if len(oidParts) != 2 {
-		return nil, errors.New("Invalid Oid type in" + oidValue)
-	}
-	if oidParts[0] != oidType {
-		return nil, errors.New("Invalid Oid type: " + oidParts[0])
-	}
-	oid := oidParts[1]
-
-	var size int64
-	sizeStr, ok := parsed["size"]
-	if !ok {
-		return nil, errors.New("Invalid Oid")
-	}
-
-	size, err = strconv.ParseInt(sizeStr, 10, 0)
 	if err != nil {
-		return nil, errors.New("Invalid size: " + sizeStr)
+		return nil, err
 	}
 
-	return NewPointer(oid, size), nil
+	value, ok = kvps["size"]
+	if !ok {
+		return nil, errors.New("Invalid size")
+	}
+
+	size, err := strconv.ParseInt(value, 10, 0)
+	if err != nil || size < 0 {
+		return nil, errors.New("Invalid size: " + value)
+	}
+
+	var extensions []PointerExtension
+	if exts != nil {
+		for key, value := range exts {
+			ext, err := parsePointerExtension(key, value)
+			if err != nil {
+				return nil, err
+			}
+			extensions = append(extensions, *ext)
+		}
+		if err = validatePointerExtensions(extensions); err != nil {
+			return nil, err
+		}
+		sort.Sort(ByPriority(extensions))
+	}
+
+	return NewPointer(oid, size, extensions), nil
 }
 
-func decodeKVData(data []byte) (map[string]string, error) {
+func parseOid(value string) (string, error) {
-	m := make(map[string]string)
+	parts := strings.SplitN(value, ":", 2)
+	if len(parts) != 2 {
+		return "", errors.New("Invalid Oid value: " + value)
+	}
+	if parts[0] != oidType {
+		return "", errors.New("Invalid Oid type: " + parts[0])
+	}
+	oid := parts[1]
+	if !oidRE.Match([]byte(oid)) {
+		return "", errors.New("Invalid Oid: " + oid)
+	}
+	return oid, nil
+}
+
+func parsePointerExtension(key string, value string) (*PointerExtension, error) {
+	keyParts := strings.SplitN(key, "-", 3)
+	if len(keyParts) != 3 || keyParts[0] != "ext" {
+		return nil, errors.New("Invalid extension value: " + value)
+	}
+
+	p, err := strconv.Atoi(keyParts[1])
+	if err != nil || p < 0 {
+		return nil, errors.New("Invalid priority: " + keyParts[1])
+	}
+
+	name := keyParts[2]
+
+	oid, err := parseOid(value)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewPointerExtension(name, p, oid), nil
+}
+
+func validatePointerExtensions(exts []PointerExtension) error {
+	m := make(map[int]struct{})
+	for _, ext := range exts {
+		if _, exist := m[ext.Priority]; exist {
+			return fmt.Errorf("Duplicate priority found: %d", ext.Priority)
+		}
+		m[ext.Priority] = struct{}{}
+	}
+	return nil
+}
+
+func decodeKVData(data []byte) (kvps map[string]string, exts map[string]string, err error) {
+	kvps = make(map[string]string)
 
 	if !matcherRE.Match(data) {
-		return m, NotAPointerError
+		err = NotAPointerError
+		return
 	}
 
 	scanner := bufio.NewScanner(bytes.NewBuffer(data))
@@ -162,23 +242,35 @@ func decodeKVData(data []byte) (map[string]string, error) {
 		}
 
 		parts := strings.SplitN(text, " ", 2)
+		if len(parts) < 2 {
+			err = fmt.Errorf("Error reading line %d: %s", line, text)
+			return
+		}
+
 		key := parts[0]
+		value := parts[1]
 
 		if numKeys <= line {
-			return m, fmt.Errorf("Extra line: %s", text)
+			err = fmt.Errorf("Extra line: %s", text)
+			return
 		}
 
 		if expected := pointerKeys[line]; key != expected {
-			return m, fmt.Errorf("Expected key %s, got %s", expected, key)
+			if !extRE.Match([]byte(key)) {
+				err = fmt.Errorf("Expected key %s, got %s", expected, key)
+				return
+			}
+			if exts == nil {
+				exts = make(map[string]string)
+			}
+			exts[key] = value
+			continue
 		}
 
 		line += 1
-		if len(parts) < 2 {
+		kvps[key] = value
-			return m, fmt.Errorf("Error reading line %d: %s", line, text)
-		}
-
-		m[key] = parts[1]
 	}
 
-	return m, scanner.Err()
+	err = scanner.Err()
+	return
 }

lfs/pointer_clean.go

@@ -45,7 +45,7 @@ func PointerClean(reader io.Reader, size int64, cb CopyCallback) (*cleanedAsset,
 	multi := io.MultiReader(bytes.NewReader(by), reader)
 	written, err := CopyWithCallback(writer, multi, size, cb)
 
-	pointer := NewPointer(hex.EncodeToString(oidHash.Sum(nil)), written)
+	pointer := NewPointer(hex.EncodeToString(oidHash.Sum(nil)), written, nil)
 	return &cleanedAsset{tmp.Name(), pointer}, err
 }
 

lfs/pointer_test.go

@@ -12,7 +12,7 @@ import (
 
 func TestEncode(t *testing.T) {
 	var buf bytes.Buffer
-	pointer := NewPointer("booya", 12345)
+	pointer := NewPointer("booya", 12345, nil)
 	_, err := EncodePointer(&buf, pointer)
 	assert.Equal(t, nil, err)
 
@@ -28,13 +28,39 @@ func TestEncode(t *testing.T) {
 	assert.Equal(t, "EOF", err.Error())
 }
 
+func TestEncodeExtensions(t *testing.T) {
+	var buf bytes.Buffer
+	exts := []PointerExtension{
+		*NewPointerExtension("foo", 0, "foo_oid"),
+		*NewPointerExtension("bar", 1, "bar_oid"),
+		*NewPointerExtension("baz", 2, "baz_oid"),
+	}
+	pointer := NewPointer("main_oid", 12345, exts)
+	_, err := EncodePointer(&buf, pointer)
+	assert.Equal(t, nil, err)
+
+	bufReader := bufio.NewReader(&buf)
+	assertLine(t, bufReader, "version https://git-lfs.github.com/spec/v1\n")
+	assertLine(t, bufReader, "ext-0-foo sha256:foo_oid\n")
+	assertLine(t, bufReader, "ext-1-bar sha256:bar_oid\n")
+	assertLine(t, bufReader, "ext-2-baz sha256:baz_oid\n")
+	assertLine(t, bufReader, "oid sha256:main_oid\n")
+	assertLine(t, bufReader, "size 12345\n")
+
+	line, err := bufReader.ReadString('\n')
+	if err == nil {
+		t.Fatalf("More to read: %s", line)
+	}
+	assert.Equal(t, "EOF", err.Error())
+}
+
 func assertLine(t *testing.T, r *bufio.Reader, expected string) {
 	actual, err := r.ReadString('\n')
 	assert.Equal(t, nil, err)
 	assert.Equal(t, expected, actual)
 }
 
-func TestLFSIniDecode(t *testing.T) {
+func TestDecode(t *testing.T) {
 	ex := `version https://git-lfs.github.com/spec/v1
 oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393
 size 12345`
@@ -43,10 +69,67 @@ size 12345`
 	assertEqualWithExample(t, ex, nil, err)
 	assertEqualWithExample(t, ex, latest, p.Version)
 	assertEqualWithExample(t, ex, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393", p.Oid)
+	assertEqualWithExample(t, ex, "sha256", p.OidType)
 	assertEqualWithExample(t, ex, int64(12345), p.Size)
 }
 
-func TestPreReleaseDecode(t *testing.T) {
+func TestDecodeExtensions(t *testing.T) {
+	ex := `version https://git-lfs.github.com/spec/v1
+ext-0-foo sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+ext-1-bar sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+ext-2-baz sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393
+size 12345`
+
+	p, err := DecodePointer(bytes.NewBufferString(ex))
+	assertEqualWithExample(t, ex, nil, err)
+	assertEqualWithExample(t, ex, latest, p.Version)
+	assertEqualWithExample(t, ex, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393", p.Oid)
+	assertEqualWithExample(t, ex, int64(12345), p.Size)
+	assertEqualWithExample(t, ex, "sha256", p.OidType)
+	assertEqualWithExample(t, ex, "foo", p.Extensions[0].Name)
+	assertEqualWithExample(t, ex, 0, p.Extensions[0].Priority)
+	assertEqualWithExample(t, ex, "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", p.Extensions[0].Oid)
+	assertEqualWithExample(t, ex, "sha256", p.Extensions[0].OidType)
+	assertEqualWithExample(t, ex, "bar", p.Extensions[1].Name)
+	assertEqualWithExample(t, ex, 1, p.Extensions[1].Priority)
+	assertEqualWithExample(t, ex, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", p.Extensions[1].Oid)
+	assertEqualWithExample(t, ex, "sha256", p.Extensions[1].OidType)
+	assertEqualWithExample(t, ex, "baz", p.Extensions[2].Name)
+	assertEqualWithExample(t, ex, 2, p.Extensions[2].Priority)
+	assertEqualWithExample(t, ex, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", p.Extensions[2].Oid)
+	assertEqualWithExample(t, ex, "sha256", p.Extensions[2].OidType)
+}
+
+func TestDecodeExtensionsSort(t *testing.T) {
+	ex := `version https://git-lfs.github.com/spec/v1
+ext-2-baz sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ext-0-foo sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+ext-1-bar sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393
+size 12345`
+
+	p, err := DecodePointer(bytes.NewBufferString(ex))
+	assertEqualWithExample(t, ex, nil, err)
+	assertEqualWithExample(t, ex, latest, p.Version)
+	assertEqualWithExample(t, ex, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393", p.Oid)
+	assertEqualWithExample(t, ex, int64(12345), p.Size)
+	assertEqualWithExample(t, ex, "sha256", p.OidType)
+	assertEqualWithExample(t, ex, "foo", p.Extensions[0].Name)
+	assertEqualWithExample(t, ex, 0, p.Extensions[0].Priority)
+	assertEqualWithExample(t, ex, "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", p.Extensions[0].Oid)
+	assertEqualWithExample(t, ex, "sha256", p.Extensions[0].OidType)
+	assertEqualWithExample(t, ex, "bar", p.Extensions[1].Name)
+	assertEqualWithExample(t, ex, 1, p.Extensions[1].Priority)
+	assertEqualWithExample(t, ex, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", p.Extensions[1].Oid)
+	assertEqualWithExample(t, ex, "sha256", p.Extensions[1].OidType)
+	assertEqualWithExample(t, ex, "baz", p.Extensions[2].Name)
+	assertEqualWithExample(t, ex, 2, p.Extensions[2].Priority)
+	assertEqualWithExample(t, ex, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", p.Extensions[2].Oid)
+	assertEqualWithExample(t, ex, "sha256", p.Extensions[2].OidType)
+}
+
+func TestDecodePreRelease(t *testing.T) {
 	ex := `version https://hawser.github.com/spec/v1
 oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393
 size 12345`
@@ -55,6 +138,7 @@ size 12345`
 	assertEqualWithExample(t, ex, nil, err)
 	assertEqualWithExample(t, ex, latest, p.Version)
 	assertEqualWithExample(t, ex, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393", p.Oid)
+	assertEqualWithExample(t, ex, "sha256", p.OidType)
 	assertEqualWithExample(t, ex, int64(12345), p.Size)
 }
 
@@ -80,6 +164,11 @@ func TestDecodeInvalid(t *testing.T) {
 		// no sha
 		"# git-media",
 
+		// bad oid
+		`version https://git-lfs.github.com/spec/v1
+oid sha256:boom
+size 12345`,
+
 		// bad oid type
 		`version https://git-lfs.github.com/spec/v1
 oid shazam:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393
@@ -127,6 +216,43 @@ wat wat`,
 		`version https://git-lfs.github.com/spec/v1
 size 12345
 oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393`,
+
+		// bad ext name
+		`version https://git-lfs.github.com/spec/v1
+ext-0-$$$$ sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393
+size 12345`,
+
+		// bad ext priority
+		`version https://git-lfs.github.com/spec/v1
+ext-#-foo sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393
+size 12345`,
+
+		// duplicate ext priority
+		`version https://git-lfs.github.com/spec/v1
+ext-0-foo sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+ext-0-bar sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393
+size 12345`,
+
+		// ext priority over 9
+		`version https://git-lfs.github.com/spec/v1
+ext-10-foo sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393
+size 12345`,
+
+		// bad ext oid
+		`version https://git-lfs.github.com/spec/v1
+ext-0-foo sha256:boom
+oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393
+size 12345`,
+
+		// bad ext oid type
+		`version https://git-lfs.github.com/spec/v1
+ext-0-foo boom:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393
+size 12345`,
 	}
 
 	for _, ex := range examples {