From 3275327b0a718282fd5bd90cfded9e529412cc64 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 5 Jun 2017 12:24:22 -0600 Subject: [PATCH 1/4] git/githistory: add 'non-repeated-subtrees' fixture --- .../fixtures/non-repeated-subtrees.git/HEAD | 1 + .../fixtures/non-repeated-subtrees.git/config | 7 +++++++ .../fixtures/non-repeated-subtrees.git/index | Bin 0 -> 248 bytes .../fixtures/non-repeated-subtrees.git/logs/HEAD | 2 ++ .../logs/refs/heads/master | 2 ++ .../07/bd7fbfc41b7d36135bcffe7c465490f4aca32d | Bin 0 -> 50 bytes .../12/7ececad475cde6da0048051d62121cabd23194 | Bin 0 -> 50 bytes .../19/acdd81ab0abc15c771fe005bf1c2825e4e6080 | Bin 0 -> 20 bytes .../37/f99c7f2706d317b3bf7ff13d574eef33d8788a | Bin 0 -> 133 bytes .../3d/1baaaceec085c52e3e57a47a75b87b7615d0ef | Bin 0 -> 82 bytes .../8d/14cbf983b3fad683171c9418998d9f68340823 | Bin 0 -> 20 bytes .../bc/63077ac5e575ccc9dbbd93dc882f1e10600ea7 | 2 ++ .../non-repeated-subtrees.git/refs/heads/master | 1 + 13 files changed, 15 insertions(+) create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/HEAD create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/config create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/index create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/logs/HEAD create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/logs/refs/heads/master create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/objects/07/bd7fbfc41b7d36135bcffe7c465490f4aca32d create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/objects/12/7ececad475cde6da0048051d62121cabd23194 create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/objects/19/acdd81ab0abc15c771fe005bf1c2825e4e6080 create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/objects/37/f99c7f2706d317b3bf7ff13d574eef33d8788a create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/objects/3d/1baaaceec085c52e3e57a47a75b87b7615d0ef create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/objects/8d/14cbf983b3fad683171c9418998d9f68340823 create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/objects/bc/63077ac5e575ccc9dbbd93dc882f1e10600ea7 create mode 100644 git/githistory/fixtures/non-repeated-subtrees.git/refs/heads/master diff --git a/git/githistory/fixtures/non-repeated-subtrees.git/HEAD b/git/githistory/fixtures/non-repeated-subtrees.git/HEAD new file mode 100644 index 00000000..cb089cd8 --- /dev/null +++ b/git/githistory/fixtures/non-repeated-subtrees.git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/git/githistory/fixtures/non-repeated-subtrees.git/config b/git/githistory/fixtures/non-repeated-subtrees.git/config new file mode 100644 index 00000000..6c9406b7 --- /dev/null +++ b/git/githistory/fixtures/non-repeated-subtrees.git/config @@ -0,0 +1,7 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true + ignorecase = true + precomposeunicode = true diff --git a/git/githistory/fixtures/non-repeated-subtrees.git/index b/git/githistory/fixtures/non-repeated-subtrees.git/index new file mode 100644 index 0000000000000000000000000000000000000000..c975ac0460505ea0acd303eb6af1384ac7c009b7 GIT binary patch literal 248 zcmZ?q402{*U|<5_NYnXSfHVV)W@KPsVKAw`%fQgMgn@zaD^N-Vh*^6@PXBD){Oek? zxXcuZnZ5HfOgNMoSQGV1DoQ}wLHbW4(-3nSIyW^g270^kjqwj)tYw)T94}4g)gZp-BDd8df`1p3xlD8 w0hdtSxl>n4&px}w;K3@JBqX!?lHru~^LX|O*7{WmSl&?W<`8u9_~>B-05&{HumAu6 literal 0 HcmV?d00001 diff --git a/git/githistory/fixtures/non-repeated-subtrees.git/logs/HEAD b/git/githistory/fixtures/non-repeated-subtrees.git/logs/HEAD new file mode 100644 index 00000000..18c82d3d --- /dev/null +++ b/git/githistory/fixtures/non-repeated-subtrees.git/logs/HEAD @@ -0,0 +1,2 @@ +0000000000000000000000000000000000000000 37f99c7f2706d317b3bf7ff13d574eef33d8788a Taylor Blau 1496686519 -0600 commit (initial): a.txt: initial commit +37f99c7f2706d317b3bf7ff13d574eef33d8788a bc63077ac5e575ccc9dbbd93dc882f1e10600ea7 Taylor Blau 1496686541 -0600 commit: subdir/b.txt: initial commit diff --git a/git/githistory/fixtures/non-repeated-subtrees.git/logs/refs/heads/master b/git/githistory/fixtures/non-repeated-subtrees.git/logs/refs/heads/master new file mode 100644 index 00000000..18c82d3d --- /dev/null +++ b/git/githistory/fixtures/non-repeated-subtrees.git/logs/refs/heads/master @@ -0,0 +1,2 @@ +0000000000000000000000000000000000000000 37f99c7f2706d317b3bf7ff13d574eef33d8788a Taylor Blau 1496686519 -0600 commit (initial): a.txt: initial commit +37f99c7f2706d317b3bf7ff13d574eef33d8788a bc63077ac5e575ccc9dbbd93dc882f1e10600ea7 Taylor Blau 1496686541 -0600 commit: subdir/b.txt: initial commit diff --git a/git/githistory/fixtures/non-repeated-subtrees.git/objects/07/bd7fbfc41b7d36135bcffe7c465490f4aca32d b/git/githistory/fixtures/non-repeated-subtrees.git/objects/07/bd7fbfc41b7d36135bcffe7c465490f4aca32d new file mode 100644 index 0000000000000000000000000000000000000000..1c3d45498744c37721f206890262d9fd5d0b484a GIT binary patch literal 50 zcmV-20L}k+0V^p=O;s>9W-v4`Ff%bxNYpE-C}HRoIsLPF^RH{o;xbbtX7v9(#xBiz{WG|Q1_1QavfkEUW Ge<1*_4-%CC literal 0 HcmV?d00001 diff --git a/git/githistory/fixtures/non-repeated-subtrees.git/objects/19/acdd81ab0abc15c771fe005bf1c2825e4e6080 b/git/githistory/fixtures/non-repeated-subtrees.git/objects/19/acdd81ab0abc15c771fe005bf1c2825e4e6080 new file mode 100644 index 0000000000000000000000000000000000000000..4aaa4e8e9de0e2f809f0b03aa32618d0dc8f3143 GIT binary patch literal 20 bcmb6V=y!@Ff%bxNYpE-C}HRoIsLPF^RH{o;xbbtX7=A,Ma@.X\b,mGb)R|Rw+ iTRQ"H1Moko~;fUWS^ q +!?:8`~OߡylNOhg3o.K \ No newline at end of file diff --git a/git/githistory/fixtures/non-repeated-subtrees.git/refs/heads/master b/git/githistory/fixtures/non-repeated-subtrees.git/refs/heads/master new file mode 100644 index 00000000..6b3e2c9f --- /dev/null +++ b/git/githistory/fixtures/non-repeated-subtrees.git/refs/heads/master @@ -0,0 +1 @@ +bc63077ac5e575ccc9dbbd93dc882f1e10600ea7 From 9dee264acb2f59c9e6cfe4d9fcb7ef313e2251e3 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 5 Jun 2017 12:25:05 -0600 Subject: [PATCH 2/4] git/githistory: add option types to NewRewriter() constructor --- git/githistory/rewriter.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/git/githistory/rewriter.go b/git/githistory/rewriter.go index 7be270dc..aacba2c2 100644 --- a/git/githistory/rewriter.go +++ b/git/githistory/rewriter.go @@ -59,15 +59,22 @@ type RewriteOptions struct { // of filepath.Join(...) or os.PathSeparator. type BlobRewriteFn func(path string, b *odb.Blob) (*odb.Blob, error) +type rewriterOption func(*Rewriter) + // NewRewriter constructs a *Rewriter from the given *ObjectDatabase instance. -func NewRewriter(db *odb.ObjectDatabase) *Rewriter { - return &Rewriter{ +func NewRewriter(db *odb.ObjectDatabase, opts ...rewriterOption) *Rewriter { + rewriter := &Rewriter{ mu: new(sync.Mutex), entries: make(map[string]*odb.TreeEntry), commits: make(map[string][]byte), db: db, } + + for _, opt := range opts { + opt(rewriter) + } + return rewriter } // Rewrite rewrites the range of commits given by *RewriteOptions.{Left,Right} From d50dc8c3ee0ab280e34a38b98acc81110db1d237 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 5 Jun 2017 12:25:36 -0600 Subject: [PATCH 3/4] git/githistory: promote assignment of path variable --- git/githistory/rewriter.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/git/githistory/rewriter.go b/git/githistory/rewriter.go index aacba2c2..d59fa5df 100644 --- a/git/githistory/rewriter.go +++ b/git/githistory/rewriter.go @@ -161,6 +161,8 @@ func (r *Rewriter) rewriteTree(sha []byte, path string, fn BlobRewriteFn) ([]byt entries := make([]*odb.TreeEntry, 0, len(tree.Entries)) for _, entry := range tree.Entries { + path := filepath.Join(path, entry.Name) + if cached := r.uncacheEntry(entry); cached != nil { entries = append(entries, cached) continue @@ -170,9 +172,9 @@ func (r *Rewriter) rewriteTree(sha []byte, path string, fn BlobRewriteFn) ([]byt switch entry.Type { case odb.BlobObjectType: - oid, err = r.rewriteBlob(entry.Oid, filepath.Join(path, entry.Name), fn) + oid, err = r.rewriteBlob(entry.Oid, path, fn) case odb.TreeObjectType: - oid, err = r.rewriteTree(entry.Oid, filepath.Join(path, entry.Name), fn) + oid, err = r.rewriteTree(entry.Oid, path, fn) default: oid = entry.Oid From 97bc74aeb81f3cd25c4cc6838a69ce62794e210e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 5 Jun 2017 12:25:58 -0600 Subject: [PATCH 4/4] git/githistory: cull out subtrees/blobs based on filter match --- git/githistory/rewriter.go | 21 +++++++++++++++++++++ git/githistory/rewriter_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/git/githistory/rewriter.go b/git/githistory/rewriter.go index d59fa5df..5192b8f6 100644 --- a/git/githistory/rewriter.go +++ b/git/githistory/rewriter.go @@ -6,6 +6,7 @@ import ( "path/filepath" "sync" + "github.com/git-lfs/git-lfs/filepathfilter" "github.com/git-lfs/git-lfs/git" "github.com/git-lfs/git-lfs/git/odb" ) @@ -22,6 +23,10 @@ type Rewriter struct { // commits is a mapping of old commit SHAs to new ones, where the ASCII // hex encoding of the SHA1 values are used as map keys. commits map[string][]byte + // filter is an optional value used to specify which tree entries + // (blobs, subtrees) are modifiable given a BlobFn. If non-nil, this + // filter will cull out any unmodifiable subtrees and blobs. + filter *filepathfilter.Filter // db is the *ObjectDatabase from which blobs, commits, and trees are // loaded from. db *odb.ObjectDatabase @@ -61,6 +66,17 @@ type BlobRewriteFn func(path string, b *odb.Blob) (*odb.Blob, error) type rewriterOption func(*Rewriter) +var ( + // WithFilter is an optional argument given to the NewRewriter + // constructor function to limit invocations of the BlobRewriteFn to + // only pathspecs that match the given *filepathfilter.Filter. + WithFilter = func(filter *filepathfilter.Filter) rewriterOption { + return func(r *Rewriter) { + r.filter = filter + } + } +) + // NewRewriter constructs a *Rewriter from the given *ObjectDatabase instance. func NewRewriter(db *odb.ObjectDatabase, opts ...rewriterOption) *Rewriter { rewriter := &Rewriter{ @@ -163,6 +179,11 @@ func (r *Rewriter) rewriteTree(sha []byte, path string, fn BlobRewriteFn) ([]byt for _, entry := range tree.Entries { path := filepath.Join(path, entry.Name) + if !r.filter.Allows(path) { + entries = append(entries, entry) + continue + } + if cached := r.uncacheEntry(entry); cached != nil { entries = append(entries, cached) continue diff --git a/git/githistory/rewriter_test.go b/git/githistory/rewriter_test.go index 6148293d..660c5b22 100644 --- a/git/githistory/rewriter_test.go +++ b/git/githistory/rewriter_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/git-lfs/git-lfs/filepathfilter" "github.com/git-lfs/git-lfs/git/odb" "github.com/stretchr/testify/assert" ) @@ -171,3 +172,27 @@ func TestRewriterVisitsUniqueEntriesWithIdenticalContents(t *testing.T) { AssertBlobContents(t, db, tree, "a.txt", "changed") AssertBlobContents(t, db, tree, "b.txt", "original") } + +func TestRewriterIgnoresPathsThatDontMatchFilter(t *testing.T) { + include := []string{"*.txt"} + exclude := []string{"subdir/**/*.txt"} + + filter := filepathfilter.New(include, exclude) + + db := DatabaseFromFixture(t, "non-repeated-subtrees.git") + r := NewRewriter(db, WithFilter(filter)) + + seen := make(map[string]int) + + _, err := r.Rewrite(&RewriteOptions{Left: "master", + BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) { + seen[path] = seen[path] + 1 + + return b, nil + }, + }) + + assert.Nil(t, err) + assert.Equal(t, 1, seen["a.txt"]) + assert.Equal(t, 0, seen["subdir/b.txt"]) +}