diff --git a/commands/command_track.go b/commands/command_track.go index fbd57061..cb46d20b 100644 --- a/commands/command_track.go +++ b/commands/command_track.go @@ -11,6 +11,7 @@ import ( "time" "github.com/git-lfs/git-lfs/git" + "github.com/git-lfs/git-lfs/git/gitattr" "github.com/git-lfs/git-lfs/tools" "github.com/spf13/cobra" ) @@ -50,9 +51,13 @@ func trackCommand(cmd *cobra.Command, args []string) { return } + mp := gitattr.NewMacroProcessor() + // Intentionally do _not_ consider global- and system-level - // .gitattributes here. - knownPatterns := git.GetAttributePaths(cfg.LocalWorkingDir(), cfg.LocalGitDir()) + // .gitattributes here. Parse them still to expand any macros. + git.GetSystemAttributePaths(mp, cfg.Os) + git.GetRootAttributePaths(mp, cfg.Git) + knownPatterns := git.GetAttributePaths(mp, cfg.LocalWorkingDir(), cfg.LocalGitDir()) lineEnd := getAttributeLineEnding(knownPatterns) if len(lineEnd) == 0 { lineEnd = gitLineEnding(cfg.Git) @@ -243,9 +248,16 @@ func listPatterns() { } func getAllKnownPatterns() []git.AttributePath { - knownPatterns := git.GetAttributePaths(cfg.LocalWorkingDir(), cfg.LocalGitDir()) - knownPatterns = append(knownPatterns, git.GetRootAttributePaths(cfg.Git)...) - knownPatterns = append(knownPatterns, git.GetSystemAttributePaths(cfg.Os)...) + mp := gitattr.NewMacroProcessor() + + // Parse these in this order so that macros in one file are properly + // expanded when referred to in a later file, then order them in the + // order we want. + systemPatterns := git.GetSystemAttributePaths(mp, cfg.Os) + globalPatterns := git.GetRootAttributePaths(mp, cfg.Git) + knownPatterns := git.GetAttributePaths(mp, cfg.LocalWorkingDir(), cfg.LocalGitDir()) + knownPatterns = append(knownPatterns, globalPatterns...) + knownPatterns = append(knownPatterns, systemPatterns...) return knownPatterns } diff --git a/git/attribs.go b/git/attribs.go index 71b3c02a..f6129898 100644 --- a/git/attribs.go +++ b/git/attribs.go @@ -1,20 +1,18 @@ package git import ( - "bufio" - "bytes" "os" "path/filepath" - "strings" "github.com/git-lfs/git-lfs/filepathfilter" + "github.com/git-lfs/git-lfs/git/gitattr" "github.com/git-lfs/git-lfs/tools" "github.com/rubyist/tracerx" ) const ( - LockableAttrib = "lockable" - FilterDisableAttrib = "-filter" + LockableAttrib = "lockable" + FilterAttrib = "filter" ) // AttributePath is a path entry in a gitattributes file which has the LFS filter @@ -34,26 +32,36 @@ type AttributeSource struct { LineEnding string } +type attrFile struct { + path string + readMacros bool +} + func (s *AttributeSource) String() string { return s.Path } // GetRootAttributePaths beahves as GetRootAttributePaths, and loads information // only from the global gitattributes file. -func GetRootAttributePaths(cfg Env) []AttributePath { - af, ok := cfg.Get("core.attributesfile") - if !ok { +func GetRootAttributePaths(mp *gitattr.MacroProcessor, cfg Env) []AttributePath { + af, _ := cfg.Get("core.attributesfile") + af, err := tools.ExpandConfigPath(af, "git/attributes") + if err != nil { + return nil + } + + if _, err := os.Stat(af); os.IsNotExist(err) { return nil } // The working directory for the root gitattributes file is blank. - return attrPaths(af, "") + return attrPaths(mp, af, "", true) } // GetSystemAttributePaths behaves as GetAttributePaths, and loads information // only from the system gitattributes file, respecting the $PREFIX environment // variable. -func GetSystemAttributePaths(env Env) []AttributePath { +func GetSystemAttributePaths(mp *gitattr.MacroProcessor, env Env) []AttributePath { prefix, _ := env.Get("PREFIX") if len(prefix) == 0 { prefix = string(filepath.Separator) @@ -65,24 +73,24 @@ func GetSystemAttributePaths(env Env) []AttributePath { return nil } - return attrPaths(path, "") + return attrPaths(mp, path, "", true) } // GetAttributePaths returns a list of entries in .gitattributes which are // configured with the filter=lfs attribute // workingDir is the root of the working copy // gitDir is the root of the git repo -func GetAttributePaths(workingDir, gitDir string) []AttributePath { +func GetAttributePaths(mp *gitattr.MacroProcessor, workingDir, gitDir string) []AttributePath { paths := make([]AttributePath, 0) - for _, path := range findAttributeFiles(workingDir, gitDir) { - paths = append(paths, attrPaths(path, workingDir)...) + for _, file := range findAttributeFiles(workingDir, gitDir) { + paths = append(paths, attrPaths(mp, file.path, workingDir, file.readMacros)...) } return paths } -func attrPaths(path, workingDir string) []AttributePath { +func attrPaths(mp *gitattr.MacroProcessor, path, workingDir string, readMacros bool) []AttributePath { attributes, err := os.Open(path) if err != nil { return nil @@ -95,55 +103,45 @@ func attrPaths(path, workingDir string) []AttributePath { reldir := filepath.Dir(relfile) source := &AttributeSource{Path: relfile} - le := &lineEndingSplitter{} - scanner := bufio.NewScanner(attributes) - scanner.Split(le.ScanLines) + lines, eol, err := gitattr.ParseLines(attributes) + if err != nil { + return nil + } - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) + lines = mp.ProcessLines(lines, readMacros) - if strings.HasPrefix(line, "#") { + for _, line := range lines { + lockable := false + tracked := false + hasFilter := false + + for _, attr := range line.Attrs { + if attr.K == FilterAttrib { + hasFilter = true + tracked = attr.V == "lfs" + } else if attr.K == LockableAttrib && attr.V == "true" { + lockable = true + } + } + + if !hasFilter && !lockable { continue } - hasFilter := strings.Contains(line, "filter=lfs") - - // Check for filter=lfs (signifying that LFS is tracking - // this file) or "lockable", which indicates that the - // file is lockable (and may or may not be tracked by - // Git LFS). - if hasFilter || - strings.Contains(line, FilterDisableAttrib) || - strings.HasSuffix(line, "lockable") { - - fields := strings.Fields(line) - pattern := fields[0] - if len(reldir) > 0 { - pattern = filepath.Join(reldir, pattern) - } - // Find lockable flag in any position after pattern to avoid - // edge case of matching "lockable" to a file pattern - lockable := false - tracked := true - for _, f := range fields[1:] { - if f == LockableAttrib { - lockable = true - } - if !hasFilter || - strings.HasPrefix(f, FilterDisableAttrib) { - tracked = false - } - } - paths = append(paths, AttributePath{ - Path: pattern, - Source: source, - Lockable: lockable, - Tracked: tracked, - }) + pattern := line.Pattern.String() + if len(reldir) > 0 { + pattern = filepath.Join(reldir, pattern) } + + paths = append(paths, AttributePath{ + Path: pattern, + Source: source, + Lockable: lockable, + Tracked: tracked, + }) } - source.LineEnding = le.LineEnding() + source.LineEnding = eol return paths } @@ -154,7 +152,7 @@ func attrPaths(path, workingDir string) []AttributePath { // workingDir is the root of the working copy // gitDir is the root of the git repo func GetAttributeFilter(workingDir, gitDir string) *filepathfilter.Filter { - paths := GetAttributePaths(workingDir, gitDir) + paths := GetAttributePaths(gitattr.NewMacroProcessor(), workingDir, gitDir) patterns := make([]filepathfilter.Pattern, 0, len(paths)) for _, path := range paths { @@ -166,53 +164,12 @@ func GetAttributeFilter(workingDir, gitDir string) *filepathfilter.Filter { return filepathfilter.NewFromPatterns(patterns, nil) } -// copies bufio.ScanLines(), counting LF vs CRLF in a file -type lineEndingSplitter struct { - LFCount int - CRLFCount int -} - -func (s *lineEndingSplitter) LineEnding() string { - if s.CRLFCount > s.LFCount { - return "\r\n" - } else if s.LFCount == 0 { - return "" - } - return "\n" -} - -func (s *lineEndingSplitter) ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF && len(data) == 0 { - return 0, nil, nil - } - if i := bytes.IndexByte(data, '\n'); i >= 0 { - // We have a full newline-terminated line. - return i + 1, s.dropCR(data[0:i]), nil - } - // If we're at EOF, we have a final, non-terminated line. Return it. - if atEOF { - return len(data), data, nil - } - // Request more data. - return 0, nil, nil -} - -// dropCR drops a terminal \r from the data. -func (s *lineEndingSplitter) dropCR(data []byte) []byte { - if len(data) > 0 && data[len(data)-1] == '\r' { - s.CRLFCount++ - return data[0 : len(data)-1] - } - s.LFCount++ - return data -} - -func findAttributeFiles(workingDir, gitDir string) []string { - var paths []string +func findAttributeFiles(workingDir, gitDir string) []attrFile { + var paths []attrFile repoAttributes := filepath.Join(gitDir, "info", "attributes") if info, err := os.Stat(repoAttributes); err == nil && !info.IsDir() { - paths = append(paths, repoAttributes) + paths = append(paths, attrFile{path: repoAttributes, readMacros: true}) } tools.FastWalkGitRepo(workingDir, func(parentDir string, info os.FileInfo, err error) { @@ -224,7 +181,11 @@ func findAttributeFiles(workingDir, gitDir string) []string { if info.IsDir() || info.Name() != ".gitattributes" { return } - paths = append(paths, filepath.Join(parentDir, info.Name())) + + paths = append(paths, attrFile{ + path: filepath.Join(parentDir, info.Name()), + readMacros: parentDir == workingDir, + }) }) // reverse the order of the files so more specific entries are found first diff --git a/git/gitattr/attr.go b/git/gitattr/attr.go index af5c4d3c..db4aa43a 100644 --- a/git/gitattr/attr.go +++ b/git/gitattr/attr.go @@ -2,6 +2,7 @@ package gitattr import ( "bufio" + "bytes" "io" "strconv" "strings" @@ -10,6 +11,8 @@ import ( "github.com/git-lfs/wildmatch" ) +const attrPrefix = "[attr]" + // Line carries a single line from a repository's .gitattributes file, affecting // a single pattern and applying zero or more attributes. type Line struct { @@ -21,6 +24,12 @@ type Line struct { // repository, while /path/to/.gitattributes affects all blobs that are // direct or indirect children of /path/to. Pattern *wildmatch.Wildmatch + // Macro is the name of a macro that, when matched, indicates that all + // of the below attributes (Attrs) should be applied to that tree + // entry. + // + // A given entry will have exactly one of Pattern or Macro set. + Macro string // Attrs is the list of attributes to be applied when the above pattern // matches a given filename. // @@ -50,12 +59,14 @@ type Attr struct { // // If an error was encountered, it will be returned and the []*Line should be // considered unusable. -func ParseLines(r io.Reader) ([]*Line, error) { +func ParseLines(r io.Reader) ([]*Line, string, error) { var lines []*Line - scanner := bufio.NewScanner(r) - for scanner.Scan() { + splitter := &lineEndingSplitter{} + scanner := bufio.NewScanner(r) + scanner.Split(splitter.ScanLines) + for scanner.Scan() { text := strings.TrimSpace(scanner.Text()) if len(text) == 0 { continue @@ -63,6 +74,7 @@ func ParseLines(r io.Reader) ([]*Line, error) { var pattern string var applied string + var macro string switch text[0] { case '#': @@ -71,17 +83,21 @@ func ParseLines(r io.Reader) ([]*Line, error) { var err error last := strings.LastIndex(text, "\"") if last == 0 { - return nil, errors.Errorf("git/gitattr: unbalanced quote: %s", text) + return nil, "", errors.Errorf("git/gitattr: unbalanced quote: %s", text) } pattern, err = strconv.Unquote(text[:last+1]) if err != nil { - return nil, errors.Wrapf(err, "git/gitattr") + return nil, "", errors.Wrapf(err, "git/gitattr") } applied = strings.TrimSpace(text[last+1:]) default: splits := strings.SplitN(text, " ", 2) - pattern = splits[0] + if strings.HasPrefix(splits[0], attrPrefix) { + macro = splits[0][len(attrPrefix):] + } else { + pattern = splits[0] + } if len(splits) == 2 { applied = splits[1] } @@ -113,16 +129,63 @@ func ParseLines(r io.Reader) ([]*Line, error) { attrs = append(attrs, &attr) } - lines = append(lines, &Line{ - Pattern: wildmatch.NewWildmatch(pattern, + var matchPattern *wildmatch.Wildmatch + if pattern != "" { + matchPattern = wildmatch.NewWildmatch(pattern, wildmatch.Basename, wildmatch.SystemCase, - ), - Attrs: attrs, + ) + } + + lines = append(lines, &Line{ + Macro: macro, + Pattern: matchPattern, + Attrs: attrs, }) } if err := scanner.Err(); err != nil { - return nil, err + return nil, "", err } - return lines, nil + return lines, splitter.LineEnding(), nil +} + +// copies bufio.ScanLines(), counting LF vs CRLF in a file +type lineEndingSplitter struct { + LFCount int + CRLFCount int +} + +func (s *lineEndingSplitter) LineEnding() string { + if s.CRLFCount > s.LFCount { + return "\r\n" + } else if s.LFCount == 0 { + return "" + } + return "\n" +} + +func (s *lineEndingSplitter) ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + if i := bytes.IndexByte(data, '\n'); i >= 0 { + // We have a full newline-terminated line. + return i + 1, s.dropCR(data[0:i]), nil + } + // If we're at EOF, we have a final, non-terminated line. Return it. + if atEOF { + return len(data), data, nil + } + // Request more data. + return 0, nil, nil +} + +// dropCR drops a terminal \r from the data. +func (s *lineEndingSplitter) dropCR(data []byte) []byte { + if len(data) > 0 && data[len(data)-1] == '\r' { + s.CRLFCount++ + return data[0 : len(data)-1] + } + s.LFCount++ + return data } diff --git a/git/gitattr/attr_test.go b/git/gitattr/attr_test.go index eaab1842..2ca90365 100644 --- a/git/gitattr/attr_test.go +++ b/git/gitattr/attr_test.go @@ -9,7 +9,7 @@ import ( ) func TestParseLines(t *testing.T) { - lines, err := ParseLines(strings.NewReader("*.dat filter=lfs")) + lines, _, err := ParseLines(strings.NewReader("*.dat filter=lfs")) assert.NoError(t, err) assert.Len(t, lines, 1) @@ -21,7 +21,7 @@ func TestParseLines(t *testing.T) { } func TestParseLinesManyAttrs(t *testing.T) { - lines, err := ParseLines(strings.NewReader( + lines, _, err := ParseLines(strings.NewReader( "*.dat filter=lfs diff=lfs merge=lfs -text crlf")) assert.NoError(t, err) @@ -38,7 +38,7 @@ func TestParseLinesManyAttrs(t *testing.T) { } func TestParseLinesManyLines(t *testing.T) { - lines, err := ParseLines(strings.NewReader(strings.Join([]string{ + lines, _, err := ParseLines(strings.NewReader(strings.Join([]string{ "*.dat filter=lfs diff=lfs merge=lfs -text", "*.jpg filter=lfs diff=lfs merge=lfs -text", "# *.pdf filter=lfs diff=lfs merge=lfs -text", @@ -76,7 +76,7 @@ func TestParseLinesManyLines(t *testing.T) { } func TestParseLinesUnset(t *testing.T) { - lines, err := ParseLines(strings.NewReader("*.dat -filter")) + lines, _, err := ParseLines(strings.NewReader("*.dat -filter")) assert.NoError(t, err) assert.Len(t, lines, 1) @@ -88,7 +88,7 @@ func TestParseLinesUnset(t *testing.T) { } func TestParseLinesUnspecified(t *testing.T) { - lines, err := ParseLines(strings.NewReader("*.dat !filter")) + lines, _, err := ParseLines(strings.NewReader("*.dat !filter")) assert.NoError(t, err) assert.Len(t, lines, 1) @@ -100,7 +100,7 @@ func TestParseLinesUnspecified(t *testing.T) { } func TestParseLinesQuotedPattern(t *testing.T) { - lines, err := ParseLines(strings.NewReader( + lines, _, err := ParseLines(strings.NewReader( "\"space *.dat\" filter=lfs")) assert.NoError(t, err) @@ -113,7 +113,7 @@ func TestParseLinesQuotedPattern(t *testing.T) { } func TestParseLinesCommented(t *testing.T) { - lines, err := ParseLines(strings.NewReader( + lines, _, err := ParseLines(strings.NewReader( "# \"space *.dat\" filter=lfs")) assert.NoError(t, err) @@ -122,7 +122,7 @@ func TestParseLinesCommented(t *testing.T) { func TestParseLinesUnbalancedQuotes(t *testing.T) { const text = "\"space *.dat filter=lfs" - lines, err := ParseLines(strings.NewReader(text)) + lines, _, err := ParseLines(strings.NewReader(text)) assert.Empty(t, lines) assert.EqualError(t, err, fmt.Sprintf( @@ -130,7 +130,7 @@ func TestParseLinesUnbalancedQuotes(t *testing.T) { } func TestParseLinesWithNoAttributes(t *testing.T) { - lines, err := ParseLines(strings.NewReader("*.dat")) + lines, _, err := ParseLines(strings.NewReader("*.dat")) assert.Len(t, lines, 1) assert.NoError(t, err) @@ -138,3 +138,31 @@ func TestParseLinesWithNoAttributes(t *testing.T) { assert.Equal(t, lines[0].Pattern.String(), "*.dat") assert.Empty(t, lines[0].Attrs) } + +func TestParseLinesWithMacros(t *testing.T) { + lines, _, err := ParseLines(strings.NewReader(strings.Join([]string{ + "[attr]lfs filter=lfs diff=lfs merge=lfs -text", + "*.dat lfs", + "*.txt text"}, "\n"))) + + assert.Len(t, lines, 3) + assert.NoError(t, err) + + assert.Equal(t, lines[0].Macro, "lfs") + assert.Nil(t, lines[0].Pattern) + assert.Len(t, lines[0].Attrs, 4) + assert.Equal(t, lines[0].Attrs[0], &Attr{K: "filter", V: "lfs"}) + assert.Equal(t, lines[0].Attrs[1], &Attr{K: "diff", V: "lfs"}) + assert.Equal(t, lines[0].Attrs[2], &Attr{K: "merge", V: "lfs"}) + assert.Equal(t, lines[0].Attrs[3], &Attr{K: "text", V: "false"}) + + assert.Equal(t, lines[1].Macro, "") + assert.Equal(t, lines[1].Pattern.String(), "*.dat") + assert.Len(t, lines[1].Attrs, 1) + assert.Equal(t, lines[1].Attrs[0], &Attr{K: "lfs", V: "true"}) + + assert.Equal(t, lines[2].Macro, "") + assert.Equal(t, lines[2].Pattern.String(), "*.txt") + assert.Len(t, lines[2].Attrs, 1) + assert.Equal(t, lines[2].Attrs[0], &Attr{K: "text", V: "true"}) +} diff --git a/git/gitattr/macro.go b/git/gitattr/macro.go new file mode 100644 index 00000000..6413ac6b --- /dev/null +++ b/git/gitattr/macro.go @@ -0,0 +1,56 @@ +package gitattr + +type MacroProcessor struct { + macros map[string][]*Attr +} + +// NewMacroProcessor returns a new MacroProcessor object for parsing macros. +func NewMacroProcessor() *MacroProcessor { + macros := make(map[string][]*Attr) + + // This is built into Git. + macros["binary"] = []*Attr{ + &Attr{K: "diff", V: "false"}, + &Attr{K: "merge", V: "false"}, + &Attr{K: "text", V: "false"}, + } + + return &MacroProcessor{ + macros: macros, + } +} + +// ProcessLines reads the specified lines, returning a new set of lines which +// all have a valid pattern. If readMacros is true, it additionally loads any +// macro lines as it reads them. +func (mp *MacroProcessor) ProcessLines(lines []*Line, readMacros bool) []*Line { + result := make([]*Line, 0, len(lines)) + for _, line := range lines { + if line.Pattern != nil { + resultLine := Line{ + Pattern: line.Pattern, + Attrs: make([]*Attr, 0, len(line.Attrs)), + } + for _, attr := range line.Attrs { + macros := mp.macros[attr.K] + if attr.V == "true" && macros != nil { + resultLine.Attrs = append( + resultLine.Attrs, + macros..., + ) + } + + // Git copies through aliases as well as + // expanding them. + resultLine.Attrs = append( + resultLine.Attrs, + attr, + ) + } + result = append(result, &resultLine) + } else if readMacros { + mp.macros[line.Macro] = line.Attrs + } + } + return result +} diff --git a/git/gitattr/macro_test.go b/git/gitattr/macro_test.go new file mode 100644 index 00000000..beccac51 --- /dev/null +++ b/git/gitattr/macro_test.go @@ -0,0 +1,126 @@ +package gitattr + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProcessLinesWithMacros(t *testing.T) { + lines, _, err := ParseLines(strings.NewReader(strings.Join([]string{ + "[attr]lfs filter=lfs diff=lfs merge=lfs -text", + "*.dat lfs", + "*.txt text"}, "\n"))) + + assert.Len(t, lines, 3) + assert.NoError(t, err) + + mp := NewMacroProcessor() + lines = mp.ProcessLines(lines, true) + + assert.Len(t, lines, 2) + + assert.Equal(t, lines[0].Macro, "") + assert.Equal(t, lines[0].Pattern.String(), "*.dat") + assert.Len(t, lines[0].Attrs, 5) + assert.Equal(t, lines[0].Attrs[0], &Attr{K: "filter", V: "lfs"}) + assert.Equal(t, lines[0].Attrs[1], &Attr{K: "diff", V: "lfs"}) + assert.Equal(t, lines[0].Attrs[2], &Attr{K: "merge", V: "lfs"}) + assert.Equal(t, lines[0].Attrs[3], &Attr{K: "text", V: "false"}) + assert.Equal(t, lines[0].Attrs[4], &Attr{K: "lfs", V: "true"}) + + assert.Equal(t, lines[1].Macro, "") + assert.Equal(t, lines[1].Pattern.String(), "*.txt") + assert.Len(t, lines[1].Attrs, 1) + assert.Equal(t, lines[1].Attrs[0], &Attr{K: "text", V: "true"}) +} + +func TestProcessLinesWithMacrosDisabled(t *testing.T) { + lines, _, err := ParseLines(strings.NewReader(strings.Join([]string{ + "[attr]lfs filter=lfs diff=lfs merge=lfs -text", + "*.dat lfs", + "*.txt text"}, "\n"))) + + assert.Len(t, lines, 3) + assert.NoError(t, err) + + mp := NewMacroProcessor() + lines = mp.ProcessLines(lines, false) + + assert.Len(t, lines, 2) + + assert.Equal(t, lines[0].Macro, "") + assert.Equal(t, lines[0].Pattern.String(), "*.dat") + assert.Len(t, lines[0].Attrs, 1) + assert.Equal(t, lines[0].Attrs[0], &Attr{K: "lfs", V: "true"}) + + assert.Equal(t, lines[1].Macro, "") + assert.Equal(t, lines[1].Pattern.String(), "*.txt") + assert.Len(t, lines[1].Attrs, 1) + assert.Equal(t, lines[1].Attrs[0], &Attr{K: "text", V: "true"}) +} + +func TestProcessLinesWithBinaryMacros(t *testing.T) { + lines, _, err := ParseLines(strings.NewReader(strings.Join([]string{ + "*.dat binary", + "*.txt text"}, "\n"))) + + assert.Len(t, lines, 2) + assert.NoError(t, err) + + mp := NewMacroProcessor() + lines = mp.ProcessLines(lines, true) + + assert.Len(t, lines, 2) + + assert.Equal(t, lines[0].Macro, "") + assert.Equal(t, lines[0].Pattern.String(), "*.dat") + assert.Len(t, lines[0].Attrs, 4) + assert.Equal(t, lines[0].Attrs[0], &Attr{K: "diff", V: "false"}) + assert.Equal(t, lines[0].Attrs[1], &Attr{K: "merge", V: "false"}) + assert.Equal(t, lines[0].Attrs[2], &Attr{K: "text", V: "false"}) + assert.Equal(t, lines[0].Attrs[3], &Attr{K: "binary", V: "true"}) + + assert.Equal(t, lines[1].Macro, "") + assert.Equal(t, lines[1].Pattern.String(), "*.txt") + assert.Len(t, lines[1].Attrs, 1) + assert.Equal(t, lines[1].Attrs[0], &Attr{K: "text", V: "true"}) +} + +func TestProcessLinesIsStateful(t *testing.T) { + lines, _, err := ParseLines(strings.NewReader(strings.Join([]string{ + "[attr]lfs filter=lfs diff=lfs merge=lfs -text", + "*.txt text"}, "\n"))) + + assert.Len(t, lines, 2) + assert.NoError(t, err) + + mp := NewMacroProcessor() + lines = mp.ProcessLines(lines, true) + + assert.Len(t, lines, 1) + + assert.Equal(t, lines[0].Macro, "") + assert.Equal(t, lines[0].Pattern.String(), "*.txt") + assert.Len(t, lines[0].Attrs, 1) + assert.Equal(t, lines[0].Attrs[0], &Attr{K: "text", V: "true"}) + + lines2, _, err := ParseLines(strings.NewReader("*.dat lfs\n")) + + assert.Len(t, lines2, 1) + assert.NoError(t, err) + + lines2 = mp.ProcessLines(lines2, false) + + assert.Len(t, lines2, 1) + + assert.Equal(t, lines2[0].Macro, "") + assert.Equal(t, lines2[0].Pattern.String(), "*.dat") + assert.Len(t, lines2[0].Attrs, 5) + assert.Equal(t, lines2[0].Attrs[0], &Attr{K: "filter", V: "lfs"}) + assert.Equal(t, lines2[0].Attrs[1], &Attr{K: "diff", V: "lfs"}) + assert.Equal(t, lines2[0].Attrs[2], &Attr{K: "merge", V: "lfs"}) + assert.Equal(t, lines2[0].Attrs[3], &Attr{K: "text", V: "false"}) + assert.Equal(t, lines2[0].Attrs[4], &Attr{K: "lfs", V: "true"}) +} diff --git a/git/gitattr/tree.go b/git/gitattr/tree.go index 45e7bde3..409b38a1 100644 --- a/git/gitattr/tree.go +++ b/git/gitattr/tree.go @@ -20,7 +20,7 @@ type Tree struct { // will be propagated up accordingly. func New(db *gitobj.ObjectDatabase, t *gitobj.Tree) (*Tree, error) { children := make(map[string]*Tree) - lines, err := linesInTree(db, t) + lines, _, err := linesInTree(db, t) if err != nil { return nil, err } @@ -59,7 +59,7 @@ func New(db *gitobj.ObjectDatabase, t *gitobj.Tree) (*Tree, error) { // linesInTree parses a given tree's .gitattributes and returns a slice of lines // in that .gitattributes, or an error. If no .gitattributes blob was found, // return nil. -func linesInTree(db *gitobj.ObjectDatabase, t *gitobj.Tree) ([]*Line, error) { +func linesInTree(db *gitobj.ObjectDatabase, t *gitobj.Tree) ([]*Line, string, error) { var at int = -1 for i, e := range t.Entries { if e.Name == ".gitattributes" { @@ -69,12 +69,12 @@ func linesInTree(db *gitobj.ObjectDatabase, t *gitobj.Tree) ([]*Line, error) { } if at < 0 { - return nil, nil + return nil, "", nil } blob, err := db.Blob(t.Entries[at].Oid) if err != nil { - return nil, err + return nil, "", err } defer blob.Close() diff --git a/go.mod b/go.mod index 83401606..ad76c04b 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ require ( github.com/alexbrainman/sspi v0.0.0-20180125232955-4729b3d4d858 github.com/git-lfs/gitobj v1.1.0 github.com/git-lfs/go-netrc v0.0.0-20180525200031-e0e9ca483a18 - github.com/git-lfs/wildmatch v1.0.0 + github.com/git-lfs/wildmatch v1.0.1 github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/kr/pty v0.0.0-20150511174710-5cf931ef8f76 github.com/mattn/go-isatty v0.0.4 diff --git a/go.sum b/go.sum index 6f83adea..81353615 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/git-lfs/gitobj v1.1.0 h1:XRUyk5nKYTWiO8U4cokO5QeoNUNBL8LKS+jXxXZdCTA= github.com/git-lfs/gitobj v1.1.0/go.mod h1:EdPNGHVxXe1jTuNXzZT1+CdJCuASoDSLPQuvNOo9nGM= github.com/git-lfs/go-netrc v0.0.0-20180525200031-e0e9ca483a18 h1:7Th0eBA4rT8WJNiM1vppjaIv9W5WJinhpbCJvRJxloI= github.com/git-lfs/go-netrc v0.0.0-20180525200031-e0e9ca483a18/go.mod h1:70O4NAtvWn1jW8V8V+OKrJJYcxDLTmIozfi2fmSz5SI= -github.com/git-lfs/wildmatch v1.0.0 h1:TKsxqSrEXWj73N4xGcN/ISal8/JJOiAcOv9LH6Zprxw= -github.com/git-lfs/wildmatch v1.0.0/go.mod h1:SdHAGnApDpnFYQ0vAxbniWR0sn7yLJ3QXo9RRfhn2ew= +github.com/git-lfs/wildmatch v1.0.1 h1:VVoPY+tqog+r8KYfi0kBvlNnGUkToqdWEPFA143EpS8= +github.com/git-lfs/wildmatch v1.0.1/go.mod h1:SdHAGnApDpnFYQ0vAxbniWR0sn7yLJ3QXo9RRfhn2ew= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/kr/pty v0.0.0-20150511174710-5cf931ef8f76 h1:i5TIRQpbCg4aJMUtVHIhkQnSw++Z405Z5pzqHqeNkdU= diff --git a/locking/lockable.go b/locking/lockable.go index 493c3a20..f4591bbc 100644 --- a/locking/lockable.go +++ b/locking/lockable.go @@ -10,6 +10,7 @@ import ( "github.com/git-lfs/git-lfs/errors" "github.com/git-lfs/git-lfs/filepathfilter" "github.com/git-lfs/git-lfs/git" + "github.com/git-lfs/git-lfs/git/gitattr" "github.com/git-lfs/git-lfs/tools" ) @@ -39,7 +40,7 @@ func (c *Client) ensureLockablesLoaded() { // Internal function to repopulate lockable patterns // You must have locked the c.lockableMutex in the caller func (c *Client) refreshLockablePatterns() { - paths := git.GetAttributePaths(c.LocalWorkingDir, c.LocalGitDir) + paths := git.GetAttributePaths(gitattr.NewMacroProcessor(), c.LocalWorkingDir, c.LocalGitDir) // Always make non-nil even if empty c.lockablePatterns = make([]string, 0, len(paths)) for _, p := range paths { diff --git a/t/t-attributes.sh b/t/t-attributes.sh new file mode 100755 index 00000000..b856ea90 --- /dev/null +++ b/t/t-attributes.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +. "$(dirname "$0")/testlib.sh" + +begin_test "macros" +( + set -e + + reponame="$(basename "$0" ".sh")" + clone_repo "$reponame" repo + + mkdir dir + printf '[attr]lfs filter=lfs diff=lfs merge=lfs -text\n*.dat lfs\n' \ + > .gitattributes + printf '[attr]lfs2 filter=lfs diff=lfs merge=lfs -text\n*.bin lfs2\n' \ + > dir/.gitattributes + git add .gitattributes dir + git commit -m 'initial import' + + contents="some data" + printf "$contents" > foo.dat + git add *.dat + git commit -m 'foo.dat' + assert_local_object "$(calc_oid "$contents")" 9 + + contents2="other data" + printf "$contents2" > dir/foo.bin + git add dir + git commit -m 'foo.bin' + refute_local_object "$(calc_oid "$contents2")" + + git lfs track '*.dat' 2>&1 | tee track.log + grep '"*.dat" already supported' track.log + + git lfs track 'dir/*.bin' 2>&1 | tee track.log + ! grep '"dir/*.bin" already supported' track.log +) +end_test + +begin_test "macros with HOME" +( + set -e + + reponame="$(basename "$0" ".sh")-home" + clone_repo "$reponame" repo-home + + mkdir -p "$HOME/.config/git" + printf '[attr]lfs filter=lfs diff=lfs merge=lfs -text\n*.dat lfs\n' \ + > "$HOME/.config/git/attributes" + + contents="some data" + printf "$contents" > foo.dat + git add *.dat + git commit -m 'foo.dat' + assert_local_object "$(calc_oid "$contents")" 9 + + git lfs track 2>&1 | tee track.log + grep '*.dat' track.log +) +end_test + +begin_test "macros with HOME split" +( + set -e + + reponame="$(basename "$0" ".sh")-home-split" + clone_repo "$reponame" repo-home-split + + mkdir -p "$HOME/.config/git" + printf '[attr]lfs filter=lfs diff=lfs merge=lfs -text\n' \ + > "$HOME/.config/git/attributes" + + printf '*.dat lfs\n' > .gitattributes + git add .gitattributes + git commit -m 'initial import' + + contents="some data" + printf "$contents" > foo.dat + git add *.dat + git commit -m 'foo.dat' + assert_local_object "$(calc_oid "$contents")" 9 + + git lfs track '*.dat' 2>&1 | tee track.log + grep '"*.dat" already supported' track.log +) +end_test diff --git a/t/t-track.sh b/t/t-track.sh index 76344b90..b3436548 100755 --- a/t/t-track.sh +++ b/t/t-track.sh @@ -47,7 +47,7 @@ begin_test "track" grep "*.jpg" .gitattributes echo "*.gif -filter -text" >> a/b/.gitattributes - echo "*.mov -filter=lfs -text" >> a/b/.gitattributes + echo "*.mov -filter -text" >> a/b/.gitattributes git lfs track | tee track.log tail -n 3 track.log | head -n 1 | grep "Listing excluded patterns" diff --git a/tools/filetools.go b/tools/filetools.go index cead7844..34ab6f61 100644 --- a/tools/filetools.go +++ b/tools/filetools.go @@ -127,7 +127,10 @@ var ( u.HomeDir = os.Getenv("HOME") return u, nil } - lookupUser func(who string) (*user.User, error) = user.Lookup + lookupUser func(who string) (*user.User, error) = user.Lookup + lookupConfigHome func() string = func() string { + return os.Getenv("XDG_CONFIG_HOME") + } ) // ExpandPath returns a copy of path with any references to the current user's @@ -179,6 +182,22 @@ func ExpandPath(path string, expand bool) (string, error) { return filepath.Join(homedir, path[len(username)+1:]), nil } +// ExpandConfigPath returns a copy of path expanded as with ExpandPath. If the +// path is empty, the default path is looked up inside $XDG_CONFIG_HOME, or +// ~/.config if that is not set. +func ExpandConfigPath(path, defaultPath string) (string, error) { + if path != "" { + return ExpandPath(path, false) + } + + configHome := lookupConfigHome() + if configHome != "" { + return filepath.Join(configHome, defaultPath), nil + } + + return ExpandPath(fmt.Sprintf("~/.config/%s", defaultPath), false) +} + // VerifyFileHash reads a file and verifies whether the SHA is correct // Returns an error if there is a problem func VerifyFileHash(oid, path string) error { diff --git a/tools/filetools_test.go b/tools/filetools_test.go index 18fdc3a6..af231ac3 100644 --- a/tools/filetools_test.go +++ b/tools/filetools_test.go @@ -121,6 +121,73 @@ func TestExpandPath(t *testing.T) { } } +type ExpandConfigPathTestCase struct { + Path string + DefaultPath string + + Want string + WantErr string + + currentUser func() (*user.User, error) + lookupConfigHome func() string +} + +func (c *ExpandConfigPathTestCase) Assert(t *testing.T) { + if c.currentUser != nil { + oldCurrentUser := currentUser + currentUser = c.currentUser + defer func() { currentUser = oldCurrentUser }() + } + + if c.lookupConfigHome != nil { + oldLookupConfigHome := lookupConfigHome + lookupConfigHome = c.lookupConfigHome + defer func() { lookupConfigHome = oldLookupConfigHome }() + } + + got, err := ExpandConfigPath(c.Path, c.DefaultPath) + if err != nil || len(c.WantErr) > 0 { + assert.EqualError(t, err, c.WantErr) + } + assert.Equal(t, filepath.ToSlash(c.Want), filepath.ToSlash(got)) +} + +func TestExpandConfigPath(t *testing.T) { + for desc, c := range map[string]*ExpandConfigPathTestCase{ + "unexpanded full path": { + Path: "/path/to/attributes", + Want: "/path/to/attributes", + }, + "expanded full path": { + Path: "~/path/to/attributes", + Want: "/home/pat/path/to/attributes", + currentUser: func() (*user.User, error) { + return &user.User{ + HomeDir: "/home/pat", + }, nil + }, + }, + "expanded default path": { + DefaultPath: "git/attributes", + Want: "/home/pat/.config/git/attributes", + currentUser: func() (*user.User, error) { + return &user.User{ + HomeDir: "/home/pat", + }, nil + }, + }, + "XDG_CONFIG_HOME set": { + DefaultPath: "git/attributes", + Want: "/home/pat/configpath/git/attributes", + lookupConfigHome: func() string { + return "/home/pat/configpath" + }, + }, + } { + t.Run(desc, c.Assert) + } +} + func TestFastWalkBasic(t *testing.T) { rootDir, err := ioutil.TempDir(os.TempDir(), "GitLfsTestFastWalkBasic") if err != nil { diff --git a/vendor/github.com/git-lfs/wildmatch/wildmatch.go b/vendor/github.com/git-lfs/wildmatch/wildmatch.go index 27f650c7..edb9990d 100644 --- a/vendor/github.com/git-lfs/wildmatch/wildmatch.go +++ b/vendor/github.com/git-lfs/wildmatch/wildmatch.go @@ -85,7 +85,7 @@ func NewWildmatch(p string, opts ...opt) *Wildmatch { const ( // escapes is a constant string containing all escapable characters - escapes = "\\[]*?" + escapes = "\\[]*?#" ) // slashEscape converts paths "p" to POSIX-compliant path, independent of which diff --git a/vendor/modules.txt b/vendor/modules.txt index a00317a4..3ff94352 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -13,7 +13,7 @@ github.com/git-lfs/gitobj/pack github.com/git-lfs/gitobj/storage # github.com/git-lfs/go-netrc v0.0.0-20180525200031-e0e9ca483a18 github.com/git-lfs/go-netrc/netrc -# github.com/git-lfs/wildmatch v1.0.0 +# github.com/git-lfs/wildmatch v1.0.1 github.com/git-lfs/wildmatch # github.com/inconshreveable/mousetrap v1.0.0 github.com/inconshreveable/mousetrap