diff --git a/commands/command_migrate.go b/commands/command_migrate.go index fe35c2f8..87139afc 100644 --- a/commands/command_migrate.go +++ b/commands/command_migrate.go @@ -43,6 +43,10 @@ var ( // migrateCommitMessage is the message to use with the commit generated // by the migrate command migrateCommitMessage string + + // exportRemote is the remote from which to download objects when + // performing an export + exportRemote string ) // migrate takes the given command and arguments, *odb.ObjectDatabase, as well @@ -297,6 +301,11 @@ func init() { importCmd.Flags().BoolVar(&migrateNoRewrite, "no-rewrite", false, "Add new history without rewriting previous") importCmd.Flags().StringVarP(&migrateCommitMessage, "message", "m", "", "With --no-rewrite, an optional commit message") + exportCmd := NewCommand("export", migrateExportCommand) + exportCmd.Flags().BoolVar(&migrateVerbose, "verbose", false, "Verbose logging") + exportCmd.Flags().StringVar(&objectMapFilePath, "object-map", "", "Object map file") + exportCmd.Flags().StringVar(&exportRemote, "remote", "", "Remote from which to download objects") + RegisterCommand("migrate", nil, func(cmd *cobra.Command) { cmd.PersistentFlags().StringVarP(&includeArg, "include", "I", "", "Include a list of paths") cmd.PersistentFlags().StringVarP(&excludeArg, "exclude", "X", "", "Exclude a list of paths") @@ -306,6 +315,6 @@ func init() { cmd.PersistentFlags().BoolVar(&migrateEverything, "everything", false, "Migrate all local references") cmd.PersistentFlags().BoolVar(&migrateSkipFetch, "skip-fetch", false, "Assume up-to-date remote references.") - cmd.AddCommand(importCmd, info) + cmd.AddCommand(exportCmd, importCmd, info) }) } diff --git a/commands/command_migrate_export.go b/commands/command_migrate_export.go new file mode 100644 index 00000000..278bed7a --- /dev/null +++ b/commands/command_migrate_export.go @@ -0,0 +1,194 @@ +package commands + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/git-lfs/git-lfs/errors" + "github.com/git-lfs/git-lfs/filepathfilter" + "github.com/git-lfs/git-lfs/git" + "github.com/git-lfs/git-lfs/git/githistory" + "github.com/git-lfs/git-lfs/git/odb" + "github.com/git-lfs/git-lfs/lfs" + "github.com/git-lfs/git-lfs/tasklog" + "github.com/git-lfs/git-lfs/tools" + "github.com/spf13/cobra" +) + +func migrateExportCommand(cmd *cobra.Command, args []string) { + l := tasklog.NewLogger(os.Stderr) + defer l.Close() + + db, err := getObjectDatabase() + if err != nil { + ExitWithError(err) + } + defer db.Close() + + rewriter := getHistoryRewriter(cmd, db, l) + + filter := rewriter.Filter() + if len(filter.Include()) <= 0 { + ExitWithError(errors.Errorf("fatal: one or more files must be specified with --include")) + } + + tracked := trackedFromExportFilter(filter) + gitfilter := lfs.NewGitFilter(cfg) + + opts := &githistory.RewriteOptions{ + Verbose: migrateVerbose, + ObjectMapFilePath: objectMapFilePath, + BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) { + if filepath.Base(path) == ".gitattributes" { + return b, nil + } + + ptr, err := lfs.DecodePointer(b.Contents) + if err != nil { + if errors.IsNotAPointerError(err) { + return b, nil + } + return nil, err + } + + downloadPath, err := gitfilter.ObjectPath(ptr.Oid) + if err != nil { + return nil, err + } + + return odb.NewBlobFromFile(downloadPath) + }, + + TreeCallbackFn: func(path string, t *odb.Tree) (*odb.Tree, error) { + if path != "/" { + // Ignore non-root trees. + return t, nil + } + + ours := tracked + theirs, err := trackedFromAttrs(db, t) + if err != nil { + return nil, err + } + + // Create a blob of the attributes that are optionally + // present in the "t" tree's .gitattributes blob, and + // union in the patterns that we've tracked. + // + // Perform this Union() operation each time we visit a + // root tree such that if the underlying .gitattributes + // is present and has a diff between commits in the + // range of commits to migrate, those changes are + // preserved. + blob, err := trackedToBlob(db, theirs.Clone().Union(ours)) + if err != nil { + return nil, err + } + + // Finally, return a copy of the tree "t" that has the + // new .gitattributes file included/replaced. + return t.Merge(&odb.TreeEntry{ + Name: ".gitattributes", + Filemode: 0100644, + Oid: blob, + }), nil + }, + + UpdateRefs: true, + } + + requireInRepo() + + opts, err = rewriteOptions(args, opts, l) + if err != nil { + ExitWithError(err) + } + + remote := cfg.Remote() + if cmd.Flag("remote").Changed { + remote = exportRemote + } + remoteURL := getAPIClient().Endpoints.RemoteEndpoint("download", remote).Url + if remoteURL == "" && cmd.Flag("remote").Changed { + ExitWithError(errors.Errorf("fatal: invalid remote %s provided", remote)) + } + + // If we have a valid remote, pre-download all objects using the Transfer Queue + if remoteURL != "" { + q := newDownloadQueue(getTransferManifestOperationRemote("Download", remote), remote) + gs := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { + if err != nil { + return + } + + if !filter.Allows(p.Name) { + return + } + + downloadPath, err := gitfilter.ObjectPath(p.Oid) + if err != nil { + return + } + + if _, err := os.Stat(downloadPath); os.IsNotExist(err) { + q.Add(p.Name, downloadPath, p.Oid, p.Size) + } + }) + gs.ScanRefs(opts.Include, opts.Exclude, nil) + + q.Wait() + + for _, err := range q.Errors() { + if err != nil { + ExitWithError(err) + } + } + } + + // Perform the rewrite + if _, err := rewriter.Rewrite(opts); err != nil { + ExitWithError(err) + } + + // Only perform `git-checkout(1) -f` if the repository is non-bare. + if bare, _ := git.IsBare(); !bare { + t := l.Waiter("migrate: checkout") + err := git.Checkout("", nil, true) + t.Complete() + + if err != nil { + ExitWithError(err) + } + } + + fetchPruneCfg := lfs.NewFetchPruneConfig(cfg.Git) + + // Set our preservation time-window for objects existing on the remote to + // 0. Because the newly rewritten commits have not yet been pushed, some + // exported objects can still exist on the remote within the time window + // and thus will not be pruned from the cache. + fetchPruneCfg.FetchRecentRefsDays = 0 + + // Prune our cache + prune(fetchPruneCfg, false, false, true) +} + +// trackedFromExportFilter returns an ordered set of strings where each entry +// is a line we intend to place in the .gitattributes file. It adds/removes the +// filter/diff/merge=lfs attributes based on patterns included/excluded in the +// given filter. Since `migrate export` removes files from Git LFS, it will +// remove attributes for included files, and add attributes for excluded files +func trackedFromExportFilter(filter *filepathfilter.Filter) *tools.OrderedSet { + tracked := tools.NewOrderedSet() + + for _, include := range filter.Include() { + tracked.Add(fmt.Sprintf("%s text !filter !merge !diff", escapeAttrPattern(include))) + } + + for _, exclude := range filter.Exclude() { + tracked.Add(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text", escapeAttrPattern(exclude))) + } + + return tracked +} diff --git a/docs/man/git-lfs-migrate.1.ronn b/docs/man/git-lfs-migrate.1.ronn index 9628156d..aff6cd4c 100644 --- a/docs/man/git-lfs-migrate.1.ronn +++ b/docs/man/git-lfs-migrate.1.ronn @@ -123,6 +123,29 @@ If `--message` is given, the new commit will be created with the provided message. If no message is given, a commit message will be generated based on the file arguments. +### EXPORT + +The 'export' mode migrates Git LFS pointer files present in the Git history out +of Git LFS, converting them into their corresponding object files. It supports +all the core 'migrate' options and these additional ones: + +* `--verbose` + Print the commit oid and filename of migrated files to STDOUT. + +* `--object-map=` + Write to 'path' a file with the mapping of each rewritten commit. The file + format is CSV with this pattern: `OLD-SHA`,`NEW-SHA` + +* `--remote=` + Download LFS objects from the provided 'git-remote' during the export. If + not provided, defaults to 'origin'. + +The 'export' mode requires at minimum a pattern provided with the `--include` +argument to specify which files to export. Files matching the `--include` +patterns will be removed from Git LFS, while files matching the `--exclude` +patterns will retain their Git LFS status. The export command will modify the +.gitattributes to set/unset any filepath patterns as given by those flags. + ## INCLUDE AND EXCLUDE You can configure Git LFS to only migrate tree entries whose pathspec matches diff --git a/git/odb/blob.go b/git/odb/blob.go index 7f9d66ae..0862b693 100644 --- a/git/odb/blob.go +++ b/git/odb/blob.go @@ -3,6 +3,9 @@ package odb import ( "bytes" "io" + "os" + + "github.com/git-lfs/git-lfs/errors" ) // Blob represents a Git object of type "blob". @@ -27,6 +30,42 @@ func NewBlobFromBytes(contents []byte) *Blob { } } +// NewBlobFromFile returns a new *Blob that contains the contents of the file +// at location "path" on disk. NewBlobFromFile does not read the file ahead of +// time, and instead defers this task until encoding the blob to the object +// database. +// +// If the file cannot be opened or stat(1)-ed, an error will be returned. +// +// When the blob receives a function call Close(), the file will also be closed, +// and any error encountered in doing so will be returned from Close(). +func NewBlobFromFile(path string) (*Blob, error) { + f, err := os.Open(path) + if err != nil { + return nil, errors.Wrapf(err, "git/odb: could not open: %s", + path) + } + + stat, err := f.Stat() + if err != nil { + return nil, errors.Wrapf(err, "git/odb: could not stat %s", + path) + } + + return &Blob{ + Contents: f, + Size: stat.Size(), + + closeFn: func() error { + if err := f.Close(); err != nil { + return errors.Wrapf(err, + "git/odb: could not close %s", path) + } + return nil + }, + }, nil +} + // Type implements Object.ObjectType by returning the correct object type for // Blobs, BlobObjectType. func (b *Blob) Type() ObjectType { return BlobObjectType } diff --git a/lfs/gitscanner.go b/lfs/gitscanner.go index 15a6d58c..3d0763a8 100644 --- a/lfs/gitscanner.go +++ b/lfs/gitscanner.go @@ -89,7 +89,20 @@ func (s *GitScanner) ScanLeftToRemote(left string, cb GitScannerFoundPointer) er } s.mu.Unlock() - return scanRefsToChan(s, callback, left, "", s.opts(ScanLeftToRemoteMode)) + return scanLeftRightToChan(s, callback, left, "", s.opts(ScanLeftToRemoteMode)) +} + +// ScanRefs through all commits reachable by refs contained in "include" and +// not reachable by any refs included in "excluded" +func (s *GitScanner) ScanRefs(include, exclude []string, cb GitScannerFoundPointer) error { + callback, err := firstGitScannerCallback(cb, s.FoundPointer) + if err != nil { + return err + } + + opts := s.opts(ScanRefsMode) + opts.SkipDeletedBlobs = false + return scanRefsToChan(s, callback, include, exclude, opts) } // ScanRefRange scans through all commits from the given left and right refs, @@ -102,7 +115,7 @@ func (s *GitScanner) ScanRefRange(left, right string, cb GitScannerFoundPointer) opts := s.opts(ScanRefsMode) opts.SkipDeletedBlobs = false - return scanRefsToChan(s, callback, left, right, opts) + return scanLeftRightToChan(s, callback, left, right, opts) } // ScanRefWithDeleted scans through all objects in the given ref, including @@ -121,7 +134,7 @@ func (s *GitScanner) ScanRef(ref string, cb GitScannerFoundPointer) error { opts := s.opts(ScanRefsMode) opts.SkipDeletedBlobs = true - return scanRefsToChan(s, callback, ref, "", opts) + return scanLeftRightToChan(s, callback, ref, "", opts) } // ScanAll scans through all objects in the git repository. @@ -133,7 +146,7 @@ func (s *GitScanner) ScanAll(cb GitScannerFoundPointer) error { opts := s.opts(ScanAllMode) opts.SkipDeletedBlobs = false - return scanRefsToChan(s, callback, "", "", opts) + return scanLeftRightToChan(s, callback, "", "", opts) } // ScanTree takes a ref and returns WrappedPointer objects in the tree at that diff --git a/lfs/gitscanner_refs.go b/lfs/gitscanner_refs.go index e66daee2..f4619e06 100644 --- a/lfs/gitscanner_refs.go +++ b/lfs/gitscanner_refs.go @@ -33,15 +33,16 @@ func (s *lockableNameSet) Check(blobSha string) (string, bool) { func noopFoundLockable(name string) {} -// scanRefsToChan takes a ref and returns a channel of WrappedPointer objects -// for all Git LFS pointers it finds for that ref. +// scanRefsToChan scans through all commits reachable by refs contained in +// "include" and not reachable by any refs included in "excluded" and returns +// a channel of WrappedPointer objects for all Git LFS pointers it finds. // Reports unique oids once only, not multiple times if >1 file uses the same content -func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, refLeft, refRight string, opt *ScanRefsOptions) error { +func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, include, exclude []string, opt *ScanRefsOptions) error { if opt == nil { panic("no scan ref options") } - revs, err := revListShas([]string{refLeft, refRight}, nil, opt) + revs, err := revListShas(include, exclude, opt) if err != nil { return err } @@ -91,6 +92,13 @@ func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, refLe return nil } +// scanLeftRightToChan takes a ref and returns a channel of WrappedPointer objects +// for all Git LFS pointers it finds for that ref. +// Reports unique oids once only, not multiple times if >1 file uses the same content +func scanLeftRightToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, refLeft, refRight string, opt *ScanRefsOptions) error { + return scanRefsToChan(scanner, pointerCb, []string{refLeft, refRight}, nil, opt) +} + // revListShas uses git rev-list to return the list of object sha1s // for the given ref. If all is true, ref is ignored. It returns a // channel from which sha1 strings can be read. diff --git a/test/test-migrate-export.sh b/test/test-migrate-export.sh new file mode 100755 index 00000000..37431bbf --- /dev/null +++ b/test/test-migrate-export.sh @@ -0,0 +1,443 @@ +#!/usr/bin/env bash + +. "test/test-migrate-fixtures.sh" +. "test/testlib.sh" + +begin_test "migrate export (default branch)" +( + set -e + + setup_multiple_local_branches_tracked + + # Add b.md, a pointer existing only on master + base64 < /dev/urandom | head -c 160 > b.md + git add b.md + git commit -m "add b.md" + + md_oid="$(calc_oid "$(cat a.md)")" + txt_oid="$(calc_oid "$(cat a.txt)")" + b_md_oid="$(calc_oid "$(cat b.md)")" + + git checkout my-feature + md_feature_oid="$(calc_oid "$(cat a.md)")" + git checkout master + + assert_pointer "refs/heads/master" "a.md" "$md_oid" "140" + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "120" + assert_pointer "refs/heads/master" "b.md" "$b_md_oid" "160" + assert_pointer "refs/heads/my-feature" "a.md" "$md_feature_oid" "30" + + git lfs migrate export --include="*.md, *.txt" + + refute_pointer "refs/heads/master" "a.md" + refute_pointer "refs/heads/master" "a.txt" + refute_pointer "refs/heads/master" "b.md" + assert_pointer "refs/heads/my-feature" "a.md" "$md_feature_oid" "30" + + # b.md should be pruned as no pointer exists to reference it + refute_local_object "$b_md_oid" "160" + + # Other objects should not be pruned as they're still referenced in `feature` + # by pointers + assert_local_object "$md_oid" "140" + assert_local_object "$txt_oid" "120" + assert_local_object "$md_feature_oid" "30" + + master="$(git rev-parse refs/heads/master)" + feature="$(git rev-parse refs/heads/my-feature)" + + master_attrs="$(git cat-file -p "$master:.gitattributes")" + feature_attrs="$(git cat-file -p "$feature:.gitattributes")" + + echo "$master_attrs" | grep -q "*.md text !filter !merge !diff" + echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff" + + [ ! $(echo "$feature_attrs" | grep -q "*.md text !filter !merge !diff") ] + [ ! $(echo "$feature_attrs" | grep -q "*.txt text !filter !merge !diff") ] +) +end_test + +begin_test "migrate export (with remote)" +( + set -e + + setup_single_remote_branch_tracked + + git push origin master + + md_oid="$(calc_oid "$(cat a.md)")" + txt_oid="$(calc_oid "$(cat a.txt)")" + + assert_pointer "refs/heads/master" "a.md" "$md_oid" "50" + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "30" + + assert_pointer "refs/remotes/origin/master" "a.md" "$md_oid" "50" + assert_pointer "refs/remotes/origin/master" "a.txt" "$txt_oid" "30" + + # Flush the cache to ensure all objects have to be downloaded + rm -rf .git/lfs/objects + + git lfs migrate export --everything --include="*.md, *.txt" + + refute_pointer "refs/heads/master" "a.md" + refute_pointer "refs/heads/master" "a.txt" + + # All pointers have been exported, so all objects should be pruned + refute_local_object "$md_oid" "50" + refute_local_object "$txt_oid" "30" + + master="$(git rev-parse refs/heads/master)" + master_attrs="$(git cat-file -p "$master:.gitattributes")" + + echo "$master_attrs" | grep -q "*.md text !filter !merge !diff" + echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff" +) +end_test + +begin_test "migrate export (include/exclude args)" +( + set -e + + setup_single_local_branch_tracked + + md_oid="$(calc_oid "$(cat a.md)")" + txt_oid="$(calc_oid "$(cat a.txt)")" + + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "120" + assert_pointer "refs/heads/master" "a.md" "$md_oid" "140" + + git lfs migrate export --include="*" --exclude="a.md" + + refute_pointer "refs/heads/master" "a.txt" + assert_pointer "refs/heads/master" "a.md" "$md_oid" "140" + + refute_local_object "$txt_oid" "120" + assert_local_object "$md_oid" "140" + + master="$(git rev-parse refs/heads/master)" + + master_attrs="$(git cat-file -p "$master:.gitattributes")" + + echo "$master_attrs" | grep -q "* text !filter !merge !diff" + echo "$master_attrs" | grep -q "a.md filter=lfs diff=lfs merge=lfs" +) +end_test + +begin_test "migrate export (bare repository)" +( + set -e + + setup_single_remote_branch_tracked + git push origin master + + md_oid="$(calc_oid "$(cat a.md)")" + txt_oid="$(calc_oid "$(cat a.txt)")" + + make_bare + + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "30" + assert_pointer "refs/heads/master" "a.md" "$md_oid" "50" + + git lfs migrate export --everything --include="*" + + refute_pointer "refs/heads/master" "a.md" + refute_pointer "refs/heads/master" "a.txt" + + # All pointers have been exported, so all objects should be pruned + refute_local_object "$md_oid" "50" + refute_local_object "$txt_oid" "30" +) +end_test + +begin_test "migrate export (given branch)" +( + set -e + + setup_multiple_local_branches_tracked + + md_oid="$(calc_oid "$(cat a.md)")" + txt_oid="$(calc_oid "$(cat a.txt)")" + + git checkout my-feature + md_feature_oid="$(calc_oid "$(cat a.md)")" + git checkout master + + assert_pointer "refs/heads/my-feature" "a.md" "$md_feature_oid" "30" + assert_pointer "refs/heads/my-feature" "a.txt" "$txt_oid" "120" + assert_pointer "refs/heads/master" "a.md" "$md_oid" "140" + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "120" + + git lfs migrate export --include="*.md,*.txt" my-feature + + refute_pointer "refs/heads/my-feature" "a.md" + refute_pointer "refs/heads/my-feature" "a.txt" + refute_pointer "refs/heads/master" "a.md" + refute_pointer "refs/heads/master" "a.txt" + + # No pointers left, so all objects should be pruned + refute_local_object "$md_feature_oid" "30" + refute_local_object "$txt_oid" "120" + refute_local_object "$md_oid" "140" + + master="$(git rev-parse refs/heads/master)" + feature="$(git rev-parse refs/heads/my-feature)" + + master_attrs="$(git cat-file -p "$master:.gitattributes")" + feature_attrs="$(git cat-file -p "$feature:.gitattributes")" + + echo "$master_attrs" | grep -q "*.md text !filter !merge !diff" + echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff" + echo "$feature_attrs" | grep -q "*.md text !filter !merge !diff" + echo "$feature_attrs" | grep -q "*.txt text !filter !merge !diff" +) +end_test + +begin_test "migrate export (no filter)" +( + set -e + + setup_multiple_local_branches_tracked + + git lfs migrate export 2>&1 | tee migrate.log + if [ ${PIPESTATUS[0]} -eq 0 ]; then + echo >&2 "fatal: expected git lfs migrate export to fail, didn't" + exit 1 + fi + + grep "fatal: one or more files must be specified with --include" migrate.log +) +end_test + +begin_test "migrate export (exclude remote refs)" +( + set -e + + setup_single_remote_branch_tracked + + md_oid="$(calc_oid "$(cat a.md)")" + txt_oid="$(calc_oid "$(cat a.txt)")" + + git checkout refs/remotes/origin/master + md_remote_oid="$(calc_oid "$(cat a.md)")" + txt_remote_oid="$(calc_oid "$(cat a.txt)")" + git checkout master + + assert_pointer "refs/heads/master" "a.md" "$md_oid" "50" + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "30" + + assert_pointer "refs/remotes/origin/master" "a.md" "$md_remote_oid" "140" + assert_pointer "refs/remotes/origin/master" "a.txt" "$txt_remote_oid" "120" + + git lfs migrate export --include="*.md,*.txt" + + refute_pointer "refs/heads/master" "a.md" + refute_pointer "refs/heads/master" "a.txt" + + refute_local_object "$md_oid" "50" + refute_local_object "$txt_oid" "30" + + assert_pointer "refs/remotes/origin/master" "a.md" "$md_remote_oid" "140" + assert_pointer "refs/remotes/origin/master" "a.txt" "$txt_remote_oid" "120" + + # Since these two objects exist on the remote, they should be removed with + # our prune operation + refute_local_object "$md_remote_oid" "140" + refute_local_object "$txt_remote_oid" "120" + + master="$(git rev-parse refs/heads/master)" + remote="$(git rev-parse refs/remotes/origin/master)" + + master_attrs="$(git cat-file -p "$master:.gitattributes")" + remote_attrs="$(git cat-file -p "$remote:.gitattributes")" + + echo "$master_attrs" | grep -q "*.md text !filter !merge !diff" + echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff" + + [ ! $(echo "$remote_attrs" | grep -q "*.md text !filter !merge !diff") ] + [ ! $(echo "$remote_attrs" | grep -q "*.txt text !filter !merge !diff") ] +) +end_test + +begin_test "migrate export (--skip-fetch)" +( + set -e + + setup_single_remote_branch_tracked + + md_master_oid="$(calc_oid "$(cat a.md)")" + txt_master_oid="$(calc_oid "$(cat a.txt)")" + + git checkout refs/remotes/origin/master + md_remote_oid="$(calc_oid "$(cat a.md)")" + txt_remote_oid="$(calc_oid "$(cat a.txt)")" + git checkout master + + git tag pseudo-remote "$(git rev-parse refs/remotes/origin/master)" + # Remove the refs/remotes/origin/master ref, and instruct 'git lfs migrate' to + # not fetch it. + git update-ref -d refs/remotes/origin/master + + assert_pointer "refs/heads/master" "a.md" "$md_master_oid" "50" + assert_pointer "pseudo-remote" "a.md" "$md_remote_oid" "140" + assert_pointer "refs/heads/master" "a.txt" "$txt_master_oid" "30" + assert_pointer "pseudo-remote" "a.txt" "$txt_remote_oid" "120" + + git lfs migrate export --skip-fetch --include="*.md,*.txt" + + refute_pointer "refs/heads/master" "a.md" + refute_pointer "pseudo-remote" "a.md" + refute_pointer "refs/heads/master" "a.txt" + refute_pointer "pseudo-remote" "a.txt" + + refute_local_object "$md_master_oid" "50" + refute_local_object "$md_remote_oid" "140" + refute_local_object "$txt_master_oid" "30" + refute_local_object "$txt_remote_oid" "120" + + master="$(git rev-parse refs/heads/master)" + remote="$(git rev-parse pseudo-remote)" + + master_attrs="$(git cat-file -p "$master:.gitattributes")" + remote_attrs="$(git cat-file -p "$remote:.gitattributes")" + + echo "$master_attrs" | grep -q "*.md text !filter !merge !diff" + echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff" + echo "$remote_attrs" | grep -q "*.md text !filter !merge !diff" + echo "$remote_attrs" | grep -q "*.txt text !filter !merge !diff" +) +end_test + +begin_test "migrate export (include/exclude ref)" +( + set -e + + setup_multiple_remote_branches_gitattrs + + md_master_oid="$(calc_oid "$(cat a.md)")" + txt_master_oid="$(calc_oid "$(cat a.txt)")" + + git checkout refs/remotes/origin/master + md_remote_oid="$(calc_oid "$(cat a.md)")" + txt_remote_oid="$(calc_oid "$(cat a.txt)")" + + git checkout my-feature + md_feature_oid="$(calc_oid "$(cat a.md)")" + txt_feature_oid="$(calc_oid "$(cat a.txt)")" + + git checkout master + + git lfs migrate export \ + --include="*.txt" \ + --include-ref=refs/heads/my-feature \ + --exclude-ref=refs/heads/master + + assert_pointer "refs/heads/master" "a.md" "$md_master_oid" "21" + assert_pointer "refs/heads/master" "a.txt" "$txt_master_oid" "20" + + assert_pointer "refs/remotes/origin/master" "a.md" "$md_remote_oid" "11" + assert_pointer "refs/remotes/origin/master" "a.txt" "$txt_remote_oid" "10" + + assert_pointer "refs/heads/my-feature" "a.md" "$md_feature_oid" "31" + refute_pointer "refs/heads/my-feature" "a.txt" + + # Master objects should not be pruned as they exist in unpushed commits + assert_local_object "$md_master_oid" "21" + assert_local_object "$txt_master_oid" "20" + + # Remote master objects should be pruned as they exist in the remote + refute_local_object "$md_remote_oid" "11" + refute_local_object "$txt_remote_oid" "10" + + # txt_feature_oid should be pruned as it's no longer a pointer, but + # md_feature_oid should remain as it's still a pointer in unpushed commits + assert_local_object "$md_feature_oid" "31" + refute_local_object "$txt_feature_oid" "30" + + master="$(git rev-parse refs/heads/master)" + feature="$(git rev-parse refs/heads/my-feature)" + remote="$(git rev-parse refs/remotes/origin/master)" + + master_attrs="$(git cat-file -p "$master:.gitattributes")" + remote_attrs="$(git cat-file -p "$remote:.gitattributes")" + feature_attrs="$(git cat-file -p "$feature:.gitattributes")" + + [ ! $(echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff") ] + [ ! $(echo "$remote_attrs" | grep -q "*.txt text !filter !merge !diff") ] + echo "$feature_attrs" | grep -q "*.txt text !filter !merge !diff" +) +end_test + +begin_test "migrate export (--object-map)" +( + set -e + + setup_multiple_local_branches_tracked + + output_dir=$(mktemp -d) + + git log --all --pretty='format:%H' > "${output_dir}/old_sha.txt" + git lfs migrate export --everything --include="*" --object-map "${output_dir}/object-map.txt" + git log --all --pretty='format:%H' > "${output_dir}/new_sha.txt" + paste -d',' "${output_dir}/old_sha.txt" "${output_dir}/new_sha.txt" > "${output_dir}/expected-map.txt" + + diff -u <(sort "${output_dir}/expected-map.txt") <(sort "${output_dir}/object-map.txt") +) +end_test + +begin_test "migrate export (--verbose)" +( + set -e + + setup_multiple_local_branches_tracked + + git lfs migrate export --everything --include="*" --verbose 2>&1 | grep -q "migrate: commit " +) +end_test + +begin_test "migrate export (--remote)" +( + set -e + + setup_single_remote_branch_tracked + + git push origin master + + md_oid="$(calc_oid "$(cat a.md)")" + txt_oid="$(calc_oid "$(cat a.txt)")" + + assert_pointer "refs/heads/master" "a.md" "$md_oid" "50" + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "30" + + # Flush the cache to ensure all objects have to be downloaded + rm -rf .git/lfs/objects + + # Setup a new remote and invalidate the default + remote_url="$(git config --get remote.origin.url)" + git remote add zeta "$remote_url" + git remote set-url origin "" + + git lfs migrate export --everything --remote="zeta" --include="*.md, *.txt" + + refute_pointer "refs/heads/master" "a.md" + refute_pointer "refs/heads/master" "a.txt" + + refute_local_object "$md_oid" "50" + refute_local_object "$txt_oid" "30" +) +end_test + +begin_test "migrate export (invalid --remote)" +( + set -e + + setup_single_remote_branch_tracked + + git lfs migrate export --include="*" --remote="zz" 2>&1 | tee migrate.log + if [ ${PIPESTATUS[0]} -eq 0 ]; then + echo >&2 "fatal: expected git lfs migrate export to fail, didn't" + exit 1 + fi + + grep "fatal: invalid remote zz provided" migrate.log +) +end_test diff --git a/test/test-migrate-fixtures.sh b/test/test-migrate-fixtures.sh index 5beee7d9..c5784100 100755 --- a/test/test-migrate-fixtures.sh +++ b/test/test-migrate-fixtures.sh @@ -84,6 +84,33 @@ setup_local_branch_with_nested_gitattrs() { git commit -m "add nested .gitattributes" } +# setup_single_local_branch_tracked creates a repository as follows: +# +# A---B +# \ +# refs/heads/master +# +# - Commit 'A' has 120, in a.txt and 140 in a.md, with both files tracked as +# pointers in Git LFS +setup_single_local_branch_tracked() { + set -e + + reponame="migrate-single-remote-branch-with-attrs" + + remove_and_create_local_repo "$reponame" + + git lfs track "*.txt" "*.md" + + git add .gitattributes + git commit -m "initial commit" + + base64 < /dev/urandom | head -c 120 > a.txt + base64 < /dev/urandom | head -c 140 > a.md + + git add a.txt a.md + git commit -m "add a.{txt,md}" +} + # setup_multiple_local_branches creates a repository as follows: # # B @@ -134,6 +161,36 @@ setup_multiple_local_branches_with_gitattrs() { git commit -m "add .gitattributes" } +# setup_multiple_local_branches_tracked creates a repo with exactly the same +# structure as in setup_multiple_local_branches, but with all files tracked by +# Git LFS +setup_multiple_local_branches_tracked() { + set -e + + reponame="migrate-info-multiple-local-branches" + + remove_and_create_local_repo "$reponame" + + git lfs track "*.txt" "*.md" + git add .gitattributes + git commit -m "initial commit" + + base64 < /dev/urandom | head -c 120 > a.txt + base64 < /dev/urandom | head -c 140 > a.md + + git add a.txt a.md + git commit -m "add a.{txt,md}" + + git checkout -b my-feature + + base64 < /dev/urandom | head -c 30 > a.md + + git add a.md + git commit -m "add an additional 30 bytes to a.md" + + git checkout master +} + # setup_local_branch_with_space creates a repository as follows: # # A @@ -201,6 +258,34 @@ setup_single_remote_branch_with_gitattrs() { git commit -m "add .gitattributes" } +# Creates a repo identical to setup_single_remote_branch, except with *.md and +# *.txt files tracked by Git LFS +setup_single_remote_branch_tracked() { + set -e + + reponame="migrate-info-single-remote-branch" + + remove_and_create_remote_repo "$reponame" + + git lfs track "*.md" "*.txt" + git add .gitattributes + git commit -m "initial commit" + + base64 < /dev/urandom | head -c 120 > a.txt + base64 < /dev/urandom | head -c 140 > a.md + + git add a.txt a.md + git commit -m "add a.{txt,md}" + + git push origin master + + base64 < /dev/urandom | head -c 30 > a.txt + base64 < /dev/urandom | head -c 50 > a.md + + git add a.md a.txt + git commit -m "add an additional 30, 50 bytes to a.{txt,md}" +} + # setup_multiple_remote_branches creates a repository as follows: # # C @@ -247,6 +332,41 @@ setup_multiple_remote_branches() { git checkout master } +# Creates a repo identical to that in setup_multiple_remote_branches(), but +# with all files tracked by Git LFS +setup_multiple_remote_branches_gitattrs() { + set -e + + reponame="migrate-info-exclude-remote-refs-given-branch" + + remove_and_create_remote_repo "$reponame" + + git lfs track "*.txt" "*.md" + git add .gitattributes + git commit -m "initial commit" + + base64 < /dev/urandom | head -c 10 > a.txt + base64 < /dev/urandom | head -c 11 > a.md + git add a.txt a.md + git commit -m "add 10, 11 bytes, a.{txt,md}" + + git push origin master + + base64 < /dev/urandom | head -c 20 > a.txt + base64 < /dev/urandom | head -c 21 > a.md + git add a.txt a.md + git commit -m "add 20, 21 bytes, a.{txt,md}" + + git checkout -b my-feature + + base64 < /dev/urandom | head -c 30 > a.txt + base64 < /dev/urandom | head -c 31 > a.md + git add a.txt a.md + git commit -m "add 30, 31 bytes, a.{txt,md}" + + git checkout master +} + # setup_single_local_branch_with_tags creates a repository as follows: # # A---B diff --git a/test/testhelpers.sh b/test/testhelpers.sh index 40180933..b8cdb1e8 100644 --- a/test/testhelpers.sh +++ b/test/testhelpers.sh @@ -24,6 +24,31 @@ assert_pointer() { fi } +# refute_pointer confirms that the file in the repository for $path in the +# given $ref is _not_ a pointer. +# +# $ refute_pointer "master" "path/to/file" +refute_pointer() { + local ref="$1" + local path="$2" + + gitblob=$(git ls-tree -lrz "$ref" | + while read -r -d $'\0' x; do + echo $x + done | + grep "$path" | cut -f 3 -d " ") + + file=$(git cat-file -p $gitblob) + version="version https://git-lfs.github.com/spec/v[0-9]" + oid="oid sha256:[0-9a-f]\{32\}" + size="size [0-9]*" + regex="$version.*$oid.*$size" + + if echo $file | grep -q "$regex"; then + exit 1 + fi +} + # assert_local_object confirms that an object file is stored for the given oid & # has the correct size # $ assert_local_object "some-oid" size