Merge pull request #3084 from git-lfs/pastelmobilesuit-migrate-export
Implement `migrate export` subcommand
This commit is contained in:
Preben Ingvaldsen
GitHub
commit
22b9113e25
@ -43,6 +43,10 @@ var (
|
||||
// migrateCommitMessage is the message to use with the commit generated
|
||||
// by the migrate command
|
||||
migrateCommitMessage string
|
||||
|
||||
// exportRemote is the remote from which to download objects when
|
||||
// performing an export
|
||||
exportRemote string
|
||||
)
|
||||
|
||||
// migrate takes the given command and arguments, *odb.ObjectDatabase, as well
|
||||
@ -297,6 +301,11 @@ func init() {
|
||||
importCmd.Flags().BoolVar(&migrateNoRewrite, "no-rewrite", false, "Add new history without rewriting previous")
|
||||
importCmd.Flags().StringVarP(&migrateCommitMessage, "message", "m", "", "With --no-rewrite, an optional commit message")
|
||||
|
||||
exportCmd := NewCommand("export", migrateExportCommand)
|
||||
exportCmd.Flags().BoolVar(&migrateVerbose, "verbose", false, "Verbose logging")
|
||||
exportCmd.Flags().StringVar(&objectMapFilePath, "object-map", "", "Object map file")
|
||||
exportCmd.Flags().StringVar(&exportRemote, "remote", "", "Remote from which to download objects")
|
||||
|
||||
RegisterCommand("migrate", nil, func(cmd *cobra.Command) {
|
||||
cmd.PersistentFlags().StringVarP(&includeArg, "include", "I", "", "Include a list of paths")
|
||||
cmd.PersistentFlags().StringVarP(&excludeArg, "exclude", "X", "", "Exclude a list of paths")
|
||||
@ -306,6 +315,6 @@ func init() {
|
||||
cmd.PersistentFlags().BoolVar(&migrateEverything, "everything", false, "Migrate all local references")
|
||||
cmd.PersistentFlags().BoolVar(&migrateSkipFetch, "skip-fetch", false, "Assume up-to-date remote references.")
|
||||
|
||||
cmd.AddCommand(importCmd, info)
|
||||
cmd.AddCommand(exportCmd, importCmd, info)
|
||||
})
|
||||
}
|
||||
|
||||
194
commands/command_migrate_export.go
Normal file
194
commands/command_migrate_export.go
Normal file
@ -0,0 +1,194 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/git-lfs/git-lfs/errors"
|
||||
"github.com/git-lfs/git-lfs/filepathfilter"
|
||||
"github.com/git-lfs/git-lfs/git"
|
||||
"github.com/git-lfs/git-lfs/git/githistory"
|
||||
"github.com/git-lfs/git-lfs/git/odb"
|
||||
"github.com/git-lfs/git-lfs/lfs"
|
||||
"github.com/git-lfs/git-lfs/tasklog"
|
||||
"github.com/git-lfs/git-lfs/tools"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func migrateExportCommand(cmd *cobra.Command, args []string) {
|
||||
l := tasklog.NewLogger(os.Stderr)
|
||||
defer l.Close()
|
||||
|
||||
db, err := getObjectDatabase()
|
||||
if err != nil {
|
||||
ExitWithError(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
rewriter := getHistoryRewriter(cmd, db, l)
|
||||
|
||||
filter := rewriter.Filter()
|
||||
if len(filter.Include()) <= 0 {
|
||||
ExitWithError(errors.Errorf("fatal: one or more files must be specified with --include"))
|
||||
}
|
||||
|
||||
tracked := trackedFromExportFilter(filter)
|
||||
gitfilter := lfs.NewGitFilter(cfg)
|
||||
|
||||
opts := &githistory.RewriteOptions{
|
||||
Verbose: migrateVerbose,
|
||||
ObjectMapFilePath: objectMapFilePath,
|
||||
BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) {
|
||||
if filepath.Base(path) == ".gitattributes" {
|
||||
return b, nil
|
||||
}
|
||||
|
||||
ptr, err := lfs.DecodePointer(b.Contents)
|
||||
if err != nil {
|
||||
if errors.IsNotAPointerError(err) {
|
||||
return b, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
downloadPath, err := gitfilter.ObjectPath(ptr.Oid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return odb.NewBlobFromFile(downloadPath)
|
||||
},
|
||||
|
||||
TreeCallbackFn: func(path string, t *odb.Tree) (*odb.Tree, error) {
|
||||
if path != "/" {
|
||||
// Ignore non-root trees.
|
||||
return t, nil
|
||||
}
|
||||
|
||||
ours := tracked
|
||||
theirs, err := trackedFromAttrs(db, t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create a blob of the attributes that are optionally
|
||||
// present in the "t" tree's .gitattributes blob, and
|
||||
// union in the patterns that we've tracked.
|
||||
//
|
||||
// Perform this Union() operation each time we visit a
|
||||
// root tree such that if the underlying .gitattributes
|
||||
// is present and has a diff between commits in the
|
||||
// range of commits to migrate, those changes are
|
||||
// preserved.
|
||||
blob, err := trackedToBlob(db, theirs.Clone().Union(ours))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Finally, return a copy of the tree "t" that has the
|
||||
// new .gitattributes file included/replaced.
|
||||
return t.Merge(&odb.TreeEntry{
|
||||
Name: ".gitattributes",
|
||||
Filemode: 0100644,
|
||||
Oid: blob,
|
||||
}), nil
|
||||
},
|
||||
|
||||
UpdateRefs: true,
|
||||
}
|
||||
|
||||
requireInRepo()
|
||||
|
||||
opts, err = rewriteOptions(args, opts, l)
|
||||
if err != nil {
|
||||
ExitWithError(err)
|
||||
}
|
||||
|
||||
remote := cfg.Remote()
|
||||
if cmd.Flag("remote").Changed {
|
||||
remote = exportRemote
|
||||
}
|
||||
remoteURL := getAPIClient().Endpoints.RemoteEndpoint("download", remote).Url
|
||||
if remoteURL == "" && cmd.Flag("remote").Changed {
|
||||
ExitWithError(errors.Errorf("fatal: invalid remote %s provided", remote))
|
||||
}
|
||||
|
||||
// If we have a valid remote, pre-download all objects using the Transfer Queue
|
||||
if remoteURL != "" {
|
||||
q := newDownloadQueue(getTransferManifestOperationRemote("Download", remote), remote)
|
||||
gs := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if !filter.Allows(p.Name) {
|
||||
return
|
||||
}
|
||||
|
||||
downloadPath, err := gitfilter.ObjectPath(p.Oid)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := os.Stat(downloadPath); os.IsNotExist(err) {
|
||||
q.Add(p.Name, downloadPath, p.Oid, p.Size)
|
||||
}
|
||||
})
|
||||
gs.ScanRefs(opts.Include, opts.Exclude, nil)
|
||||
|
||||
q.Wait()
|
||||
|
||||
for _, err := range q.Errors() {
|
||||
if err != nil {
|
||||
ExitWithError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Perform the rewrite
|
||||
if _, err := rewriter.Rewrite(opts); err != nil {
|
||||
ExitWithError(err)
|
||||
}
|
||||
|
||||
// Only perform `git-checkout(1) -f` if the repository is non-bare.
|
||||
if bare, _ := git.IsBare(); !bare {
|
||||
t := l.Waiter("migrate: checkout")
|
||||
err := git.Checkout("", nil, true)
|
||||
t.Complete()
|
||||
|
||||
if err != nil {
|
||||
ExitWithError(err)
|
||||
}
|
||||
}
|
||||
|
||||
fetchPruneCfg := lfs.NewFetchPruneConfig(cfg.Git)
|
||||
|
||||
// Set our preservation time-window for objects existing on the remote to
|
||||
// 0. Because the newly rewritten commits have not yet been pushed, some
|
||||
// exported objects can still exist on the remote within the time window
|
||||
// and thus will not be pruned from the cache.
|
||||
fetchPruneCfg.FetchRecentRefsDays = 0
|
||||
|
||||
// Prune our cache
|
||||
prune(fetchPruneCfg, false, false, true)
|
||||
}
|
||||
|
||||
// trackedFromExportFilter returns an ordered set of strings where each entry
|
||||
// is a line we intend to place in the .gitattributes file. It adds/removes the
|
||||
// filter/diff/merge=lfs attributes based on patterns included/excluded in the
|
||||
// given filter. Since `migrate export` removes files from Git LFS, it will
|
||||
// remove attributes for included files, and add attributes for excluded files
|
||||
func trackedFromExportFilter(filter *filepathfilter.Filter) *tools.OrderedSet {
|
||||
tracked := tools.NewOrderedSet()
|
||||
|
||||
for _, include := range filter.Include() {
|
||||
tracked.Add(fmt.Sprintf("%s text !filter !merge !diff", escapeAttrPattern(include)))
|
||||
}
|
||||
|
||||
for _, exclude := range filter.Exclude() {
|
||||
tracked.Add(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text", escapeAttrPattern(exclude)))
|
||||
}
|
||||
|
||||
return tracked
|
||||
}
|
||||
@ -123,6 +123,29 @@ If `--message` is given, the new commit will be created with the provided
|
||||
message. If no message is given, a commit message will be generated based on the
|
||||
file arguments.
|
||||
|
||||
### EXPORT
|
||||
|
||||
The 'export' mode migrates Git LFS pointer files present in the Git history out
|
||||
of Git LFS, converting them into their corresponding object files. It supports
|
||||
all the core 'migrate' options and these additional ones:
|
||||
|
||||
* `--verbose`
|
||||
Print the commit oid and filename of migrated files to STDOUT.
|
||||
|
||||
* `--object-map=<path>`
|
||||
Write to 'path' a file with the mapping of each rewritten commit. The file
|
||||
format is CSV with this pattern: `OLD-SHA`,`NEW-SHA`
|
||||
|
||||
* `--remote=<git-remote>`
|
||||
Download LFS objects from the provided 'git-remote' during the export. If
|
||||
not provided, defaults to 'origin'.
|
||||
|
||||
The 'export' mode requires at minimum a pattern provided with the `--include`
|
||||
argument to specify which files to export. Files matching the `--include`
|
||||
patterns will be removed from Git LFS, while files matching the `--exclude`
|
||||
patterns will retain their Git LFS status. The export command will modify the
|
||||
.gitattributes to set/unset any filepath patterns as given by those flags.
|
||||
|
||||
## INCLUDE AND EXCLUDE
|
||||
|
||||
You can configure Git LFS to only migrate tree entries whose pathspec matches
|
||||
|
||||
@ -3,6 +3,9 @@ package odb
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/git-lfs/git-lfs/errors"
|
||||
)
|
||||
|
||||
// Blob represents a Git object of type "blob".
|
||||
@ -27,6 +30,42 @@ func NewBlobFromBytes(contents []byte) *Blob {
|
||||
}
|
||||
}
|
||||
|
||||
// NewBlobFromFile returns a new *Blob that contains the contents of the file
|
||||
// at location "path" on disk. NewBlobFromFile does not read the file ahead of
|
||||
// time, and instead defers this task until encoding the blob to the object
|
||||
// database.
|
||||
//
|
||||
// If the file cannot be opened or stat(1)-ed, an error will be returned.
|
||||
//
|
||||
// When the blob receives a function call Close(), the file will also be closed,
|
||||
// and any error encountered in doing so will be returned from Close().
|
||||
func NewBlobFromFile(path string) (*Blob, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "git/odb: could not open: %s",
|
||||
path)
|
||||
}
|
||||
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "git/odb: could not stat %s",
|
||||
path)
|
||||
}
|
||||
|
||||
return &Blob{
|
||||
Contents: f,
|
||||
Size: stat.Size(),
|
||||
|
||||
closeFn: func() error {
|
||||
if err := f.Close(); err != nil {
|
||||
return errors.Wrapf(err,
|
||||
"git/odb: could not close %s", path)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Type implements Object.ObjectType by returning the correct object type for
|
||||
// Blobs, BlobObjectType.
|
||||
func (b *Blob) Type() ObjectType { return BlobObjectType }
|
||||
|
||||
@ -89,7 +89,20 @@ func (s *GitScanner) ScanLeftToRemote(left string, cb GitScannerFoundPointer) er
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
return scanRefsToChan(s, callback, left, "", s.opts(ScanLeftToRemoteMode))
|
||||
return scanLeftRightToChan(s, callback, left, "", s.opts(ScanLeftToRemoteMode))
|
||||
}
|
||||
|
||||
// ScanRefs through all commits reachable by refs contained in "include" and
|
||||
// not reachable by any refs included in "excluded"
|
||||
func (s *GitScanner) ScanRefs(include, exclude []string, cb GitScannerFoundPointer) error {
|
||||
callback, err := firstGitScannerCallback(cb, s.FoundPointer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
opts := s.opts(ScanRefsMode)
|
||||
opts.SkipDeletedBlobs = false
|
||||
return scanRefsToChan(s, callback, include, exclude, opts)
|
||||
}
|
||||
|
||||
// ScanRefRange scans through all commits from the given left and right refs,
|
||||
@ -102,7 +115,7 @@ func (s *GitScanner) ScanRefRange(left, right string, cb GitScannerFoundPointer)
|
||||
|
||||
opts := s.opts(ScanRefsMode)
|
||||
opts.SkipDeletedBlobs = false
|
||||
return scanRefsToChan(s, callback, left, right, opts)
|
||||
return scanLeftRightToChan(s, callback, left, right, opts)
|
||||
}
|
||||
|
||||
// ScanRefWithDeleted scans through all objects in the given ref, including
|
||||
@ -121,7 +134,7 @@ func (s *GitScanner) ScanRef(ref string, cb GitScannerFoundPointer) error {
|
||||
|
||||
opts := s.opts(ScanRefsMode)
|
||||
opts.SkipDeletedBlobs = true
|
||||
return scanRefsToChan(s, callback, ref, "", opts)
|
||||
return scanLeftRightToChan(s, callback, ref, "", opts)
|
||||
}
|
||||
|
||||
// ScanAll scans through all objects in the git repository.
|
||||
@ -133,7 +146,7 @@ func (s *GitScanner) ScanAll(cb GitScannerFoundPointer) error {
|
||||
|
||||
opts := s.opts(ScanAllMode)
|
||||
opts.SkipDeletedBlobs = false
|
||||
return scanRefsToChan(s, callback, "", "", opts)
|
||||
return scanLeftRightToChan(s, callback, "", "", opts)
|
||||
}
|
||||
|
||||
// ScanTree takes a ref and returns WrappedPointer objects in the tree at that
|
||||
|
||||
@ -33,15 +33,16 @@ func (s *lockableNameSet) Check(blobSha string) (string, bool) {
|
||||
|
||||
func noopFoundLockable(name string) {}
|
||||
|
||||
// scanRefsToChan takes a ref and returns a channel of WrappedPointer objects
|
||||
// for all Git LFS pointers it finds for that ref.
|
||||
// scanRefsToChan scans through all commits reachable by refs contained in
|
||||
// "include" and not reachable by any refs included in "excluded" and returns
|
||||
// a channel of WrappedPointer objects for all Git LFS pointers it finds.
|
||||
// Reports unique oids once only, not multiple times if >1 file uses the same content
|
||||
func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, refLeft, refRight string, opt *ScanRefsOptions) error {
|
||||
func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, include, exclude []string, opt *ScanRefsOptions) error {
|
||||
if opt == nil {
|
||||
panic("no scan ref options")
|
||||
}
|
||||
|
||||
revs, err := revListShas([]string{refLeft, refRight}, nil, opt)
|
||||
revs, err := revListShas(include, exclude, opt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -91,6 +92,13 @@ func scanRefsToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, refLe
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanLeftRightToChan takes a ref and returns a channel of WrappedPointer objects
|
||||
// for all Git LFS pointers it finds for that ref.
|
||||
// Reports unique oids once only, not multiple times if >1 file uses the same content
|
||||
func scanLeftRightToChan(scanner *GitScanner, pointerCb GitScannerFoundPointer, refLeft, refRight string, opt *ScanRefsOptions) error {
|
||||
return scanRefsToChan(scanner, pointerCb, []string{refLeft, refRight}, nil, opt)
|
||||
}
|
||||
|
||||
// revListShas uses git rev-list to return the list of object sha1s
|
||||
// for the given ref. If all is true, ref is ignored. It returns a
|
||||
// channel from which sha1 strings can be read.
|
||||
|
||||
443
test/test-migrate-export.sh
Executable file
443
test/test-migrate-export.sh
Executable file
@ -0,0 +1,443 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
. "test/test-migrate-fixtures.sh"
|
||||
. "test/testlib.sh"
|
||||
|
||||
begin_test "migrate export (default branch)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_multiple_local_branches_tracked
|
||||
|
||||
# Add b.md, a pointer existing only on master
|
||||
base64 < /dev/urandom | head -c 160 > b.md
|
||||
git add b.md
|
||||
git commit -m "add b.md"
|
||||
|
||||
md_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_oid="$(calc_oid "$(cat a.txt)")"
|
||||
b_md_oid="$(calc_oid "$(cat b.md)")"
|
||||
|
||||
git checkout my-feature
|
||||
md_feature_oid="$(calc_oid "$(cat a.md)")"
|
||||
git checkout master
|
||||
|
||||
assert_pointer "refs/heads/master" "a.md" "$md_oid" "140"
|
||||
assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "120"
|
||||
assert_pointer "refs/heads/master" "b.md" "$b_md_oid" "160"
|
||||
assert_pointer "refs/heads/my-feature" "a.md" "$md_feature_oid" "30"
|
||||
|
||||
git lfs migrate export --include="*.md, *.txt"
|
||||
|
||||
refute_pointer "refs/heads/master" "a.md"
|
||||
refute_pointer "refs/heads/master" "a.txt"
|
||||
refute_pointer "refs/heads/master" "b.md"
|
||||
assert_pointer "refs/heads/my-feature" "a.md" "$md_feature_oid" "30"
|
||||
|
||||
# b.md should be pruned as no pointer exists to reference it
|
||||
refute_local_object "$b_md_oid" "160"
|
||||
|
||||
# Other objects should not be pruned as they're still referenced in `feature`
|
||||
# by pointers
|
||||
assert_local_object "$md_oid" "140"
|
||||
assert_local_object "$txt_oid" "120"
|
||||
assert_local_object "$md_feature_oid" "30"
|
||||
|
||||
master="$(git rev-parse refs/heads/master)"
|
||||
feature="$(git rev-parse refs/heads/my-feature)"
|
||||
|
||||
master_attrs="$(git cat-file -p "$master:.gitattributes")"
|
||||
feature_attrs="$(git cat-file -p "$feature:.gitattributes")"
|
||||
|
||||
echo "$master_attrs" | grep -q "*.md text !filter !merge !diff"
|
||||
echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff"
|
||||
|
||||
[ ! $(echo "$feature_attrs" | grep -q "*.md text !filter !merge !diff") ]
|
||||
[ ! $(echo "$feature_attrs" | grep -q "*.txt text !filter !merge !diff") ]
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (with remote)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_single_remote_branch_tracked
|
||||
|
||||
git push origin master
|
||||
|
||||
md_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_oid="$(calc_oid "$(cat a.txt)")"
|
||||
|
||||
assert_pointer "refs/heads/master" "a.md" "$md_oid" "50"
|
||||
assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "30"
|
||||
|
||||
assert_pointer "refs/remotes/origin/master" "a.md" "$md_oid" "50"
|
||||
assert_pointer "refs/remotes/origin/master" "a.txt" "$txt_oid" "30"
|
||||
|
||||
# Flush the cache to ensure all objects have to be downloaded
|
||||
rm -rf .git/lfs/objects
|
||||
|
||||
git lfs migrate export --everything --include="*.md, *.txt"
|
||||
|
||||
refute_pointer "refs/heads/master" "a.md"
|
||||
refute_pointer "refs/heads/master" "a.txt"
|
||||
|
||||
# All pointers have been exported, so all objects should be pruned
|
||||
refute_local_object "$md_oid" "50"
|
||||
refute_local_object "$txt_oid" "30"
|
||||
|
||||
master="$(git rev-parse refs/heads/master)"
|
||||
master_attrs="$(git cat-file -p "$master:.gitattributes")"
|
||||
|
||||
echo "$master_attrs" | grep -q "*.md text !filter !merge !diff"
|
||||
echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff"
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (include/exclude args)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_single_local_branch_tracked
|
||||
|
||||
md_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_oid="$(calc_oid "$(cat a.txt)")"
|
||||
|
||||
assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "120"
|
||||
assert_pointer "refs/heads/master" "a.md" "$md_oid" "140"
|
||||
|
||||
git lfs migrate export --include="*" --exclude="a.md"
|
||||
|
||||
refute_pointer "refs/heads/master" "a.txt"
|
||||
assert_pointer "refs/heads/master" "a.md" "$md_oid" "140"
|
||||
|
||||
refute_local_object "$txt_oid" "120"
|
||||
assert_local_object "$md_oid" "140"
|
||||
|
||||
master="$(git rev-parse refs/heads/master)"
|
||||
|
||||
master_attrs="$(git cat-file -p "$master:.gitattributes")"
|
||||
|
||||
echo "$master_attrs" | grep -q "* text !filter !merge !diff"
|
||||
echo "$master_attrs" | grep -q "a.md filter=lfs diff=lfs merge=lfs"
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (bare repository)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_single_remote_branch_tracked
|
||||
git push origin master
|
||||
|
||||
md_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_oid="$(calc_oid "$(cat a.txt)")"
|
||||
|
||||
make_bare
|
||||
|
||||
assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "30"
|
||||
assert_pointer "refs/heads/master" "a.md" "$md_oid" "50"
|
||||
|
||||
git lfs migrate export --everything --include="*"
|
||||
|
||||
refute_pointer "refs/heads/master" "a.md"
|
||||
refute_pointer "refs/heads/master" "a.txt"
|
||||
|
||||
# All pointers have been exported, so all objects should be pruned
|
||||
refute_local_object "$md_oid" "50"
|
||||
refute_local_object "$txt_oid" "30"
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (given branch)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_multiple_local_branches_tracked
|
||||
|
||||
md_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_oid="$(calc_oid "$(cat a.txt)")"
|
||||
|
||||
git checkout my-feature
|
||||
md_feature_oid="$(calc_oid "$(cat a.md)")"
|
||||
git checkout master
|
||||
|
||||
assert_pointer "refs/heads/my-feature" "a.md" "$md_feature_oid" "30"
|
||||
assert_pointer "refs/heads/my-feature" "a.txt" "$txt_oid" "120"
|
||||
assert_pointer "refs/heads/master" "a.md" "$md_oid" "140"
|
||||
assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "120"
|
||||
|
||||
git lfs migrate export --include="*.md,*.txt" my-feature
|
||||
|
||||
refute_pointer "refs/heads/my-feature" "a.md"
|
||||
refute_pointer "refs/heads/my-feature" "a.txt"
|
||||
refute_pointer "refs/heads/master" "a.md"
|
||||
refute_pointer "refs/heads/master" "a.txt"
|
||||
|
||||
# No pointers left, so all objects should be pruned
|
||||
refute_local_object "$md_feature_oid" "30"
|
||||
refute_local_object "$txt_oid" "120"
|
||||
refute_local_object "$md_oid" "140"
|
||||
|
||||
master="$(git rev-parse refs/heads/master)"
|
||||
feature="$(git rev-parse refs/heads/my-feature)"
|
||||
|
||||
master_attrs="$(git cat-file -p "$master:.gitattributes")"
|
||||
feature_attrs="$(git cat-file -p "$feature:.gitattributes")"
|
||||
|
||||
echo "$master_attrs" | grep -q "*.md text !filter !merge !diff"
|
||||
echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff"
|
||||
echo "$feature_attrs" | grep -q "*.md text !filter !merge !diff"
|
||||
echo "$feature_attrs" | grep -q "*.txt text !filter !merge !diff"
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (no filter)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_multiple_local_branches_tracked
|
||||
|
||||
git lfs migrate export 2>&1 | tee migrate.log
|
||||
if [ ${PIPESTATUS[0]} -eq 0 ]; then
|
||||
echo >&2 "fatal: expected git lfs migrate export to fail, didn't"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
grep "fatal: one or more files must be specified with --include" migrate.log
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (exclude remote refs)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_single_remote_branch_tracked
|
||||
|
||||
md_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_oid="$(calc_oid "$(cat a.txt)")"
|
||||
|
||||
git checkout refs/remotes/origin/master
|
||||
md_remote_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_remote_oid="$(calc_oid "$(cat a.txt)")"
|
||||
git checkout master
|
||||
|
||||
assert_pointer "refs/heads/master" "a.md" "$md_oid" "50"
|
||||
assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "30"
|
||||
|
||||
assert_pointer "refs/remotes/origin/master" "a.md" "$md_remote_oid" "140"
|
||||
assert_pointer "refs/remotes/origin/master" "a.txt" "$txt_remote_oid" "120"
|
||||
|
||||
git lfs migrate export --include="*.md,*.txt"
|
||||
|
||||
refute_pointer "refs/heads/master" "a.md"
|
||||
refute_pointer "refs/heads/master" "a.txt"
|
||||
|
||||
refute_local_object "$md_oid" "50"
|
||||
refute_local_object "$txt_oid" "30"
|
||||
|
||||
assert_pointer "refs/remotes/origin/master" "a.md" "$md_remote_oid" "140"
|
||||
assert_pointer "refs/remotes/origin/master" "a.txt" "$txt_remote_oid" "120"
|
||||
|
||||
# Since these two objects exist on the remote, they should be removed with
|
||||
# our prune operation
|
||||
refute_local_object "$md_remote_oid" "140"
|
||||
refute_local_object "$txt_remote_oid" "120"
|
||||
|
||||
master="$(git rev-parse refs/heads/master)"
|
||||
remote="$(git rev-parse refs/remotes/origin/master)"
|
||||
|
||||
master_attrs="$(git cat-file -p "$master:.gitattributes")"
|
||||
remote_attrs="$(git cat-file -p "$remote:.gitattributes")"
|
||||
|
||||
echo "$master_attrs" | grep -q "*.md text !filter !merge !diff"
|
||||
echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff"
|
||||
|
||||
[ ! $(echo "$remote_attrs" | grep -q "*.md text !filter !merge !diff") ]
|
||||
[ ! $(echo "$remote_attrs" | grep -q "*.txt text !filter !merge !diff") ]
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (--skip-fetch)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_single_remote_branch_tracked
|
||||
|
||||
md_master_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_master_oid="$(calc_oid "$(cat a.txt)")"
|
||||
|
||||
git checkout refs/remotes/origin/master
|
||||
md_remote_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_remote_oid="$(calc_oid "$(cat a.txt)")"
|
||||
git checkout master
|
||||
|
||||
git tag pseudo-remote "$(git rev-parse refs/remotes/origin/master)"
|
||||
# Remove the refs/remotes/origin/master ref, and instruct 'git lfs migrate' to
|
||||
# not fetch it.
|
||||
git update-ref -d refs/remotes/origin/master
|
||||
|
||||
assert_pointer "refs/heads/master" "a.md" "$md_master_oid" "50"
|
||||
assert_pointer "pseudo-remote" "a.md" "$md_remote_oid" "140"
|
||||
assert_pointer "refs/heads/master" "a.txt" "$txt_master_oid" "30"
|
||||
assert_pointer "pseudo-remote" "a.txt" "$txt_remote_oid" "120"
|
||||
|
||||
git lfs migrate export --skip-fetch --include="*.md,*.txt"
|
||||
|
||||
refute_pointer "refs/heads/master" "a.md"
|
||||
refute_pointer "pseudo-remote" "a.md"
|
||||
refute_pointer "refs/heads/master" "a.txt"
|
||||
refute_pointer "pseudo-remote" "a.txt"
|
||||
|
||||
refute_local_object "$md_master_oid" "50"
|
||||
refute_local_object "$md_remote_oid" "140"
|
||||
refute_local_object "$txt_master_oid" "30"
|
||||
refute_local_object "$txt_remote_oid" "120"
|
||||
|
||||
master="$(git rev-parse refs/heads/master)"
|
||||
remote="$(git rev-parse pseudo-remote)"
|
||||
|
||||
master_attrs="$(git cat-file -p "$master:.gitattributes")"
|
||||
remote_attrs="$(git cat-file -p "$remote:.gitattributes")"
|
||||
|
||||
echo "$master_attrs" | grep -q "*.md text !filter !merge !diff"
|
||||
echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff"
|
||||
echo "$remote_attrs" | grep -q "*.md text !filter !merge !diff"
|
||||
echo "$remote_attrs" | grep -q "*.txt text !filter !merge !diff"
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (include/exclude ref)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_multiple_remote_branches_gitattrs
|
||||
|
||||
md_master_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_master_oid="$(calc_oid "$(cat a.txt)")"
|
||||
|
||||
git checkout refs/remotes/origin/master
|
||||
md_remote_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_remote_oid="$(calc_oid "$(cat a.txt)")"
|
||||
|
||||
git checkout my-feature
|
||||
md_feature_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_feature_oid="$(calc_oid "$(cat a.txt)")"
|
||||
|
||||
git checkout master
|
||||
|
||||
git lfs migrate export \
|
||||
--include="*.txt" \
|
||||
--include-ref=refs/heads/my-feature \
|
||||
--exclude-ref=refs/heads/master
|
||||
|
||||
assert_pointer "refs/heads/master" "a.md" "$md_master_oid" "21"
|
||||
assert_pointer "refs/heads/master" "a.txt" "$txt_master_oid" "20"
|
||||
|
||||
assert_pointer "refs/remotes/origin/master" "a.md" "$md_remote_oid" "11"
|
||||
assert_pointer "refs/remotes/origin/master" "a.txt" "$txt_remote_oid" "10"
|
||||
|
||||
assert_pointer "refs/heads/my-feature" "a.md" "$md_feature_oid" "31"
|
||||
refute_pointer "refs/heads/my-feature" "a.txt"
|
||||
|
||||
# Master objects should not be pruned as they exist in unpushed commits
|
||||
assert_local_object "$md_master_oid" "21"
|
||||
assert_local_object "$txt_master_oid" "20"
|
||||
|
||||
# Remote master objects should be pruned as they exist in the remote
|
||||
refute_local_object "$md_remote_oid" "11"
|
||||
refute_local_object "$txt_remote_oid" "10"
|
||||
|
||||
# txt_feature_oid should be pruned as it's no longer a pointer, but
|
||||
# md_feature_oid should remain as it's still a pointer in unpushed commits
|
||||
assert_local_object "$md_feature_oid" "31"
|
||||
refute_local_object "$txt_feature_oid" "30"
|
||||
|
||||
master="$(git rev-parse refs/heads/master)"
|
||||
feature="$(git rev-parse refs/heads/my-feature)"
|
||||
remote="$(git rev-parse refs/remotes/origin/master)"
|
||||
|
||||
master_attrs="$(git cat-file -p "$master:.gitattributes")"
|
||||
remote_attrs="$(git cat-file -p "$remote:.gitattributes")"
|
||||
feature_attrs="$(git cat-file -p "$feature:.gitattributes")"
|
||||
|
||||
[ ! $(echo "$master_attrs" | grep -q "*.txt text !filter !merge !diff") ]
|
||||
[ ! $(echo "$remote_attrs" | grep -q "*.txt text !filter !merge !diff") ]
|
||||
echo "$feature_attrs" | grep -q "*.txt text !filter !merge !diff"
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (--object-map)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_multiple_local_branches_tracked
|
||||
|
||||
output_dir=$(mktemp -d)
|
||||
|
||||
git log --all --pretty='format:%H' > "${output_dir}/old_sha.txt"
|
||||
git lfs migrate export --everything --include="*" --object-map "${output_dir}/object-map.txt"
|
||||
git log --all --pretty='format:%H' > "${output_dir}/new_sha.txt"
|
||||
paste -d',' "${output_dir}/old_sha.txt" "${output_dir}/new_sha.txt" > "${output_dir}/expected-map.txt"
|
||||
|
||||
diff -u <(sort "${output_dir}/expected-map.txt") <(sort "${output_dir}/object-map.txt")
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (--verbose)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_multiple_local_branches_tracked
|
||||
|
||||
git lfs migrate export --everything --include="*" --verbose 2>&1 | grep -q "migrate: commit "
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (--remote)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_single_remote_branch_tracked
|
||||
|
||||
git push origin master
|
||||
|
||||
md_oid="$(calc_oid "$(cat a.md)")"
|
||||
txt_oid="$(calc_oid "$(cat a.txt)")"
|
||||
|
||||
assert_pointer "refs/heads/master" "a.md" "$md_oid" "50"
|
||||
assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "30"
|
||||
|
||||
# Flush the cache to ensure all objects have to be downloaded
|
||||
rm -rf .git/lfs/objects
|
||||
|
||||
# Setup a new remote and invalidate the default
|
||||
remote_url="$(git config --get remote.origin.url)"
|
||||
git remote add zeta "$remote_url"
|
||||
git remote set-url origin ""
|
||||
|
||||
git lfs migrate export --everything --remote="zeta" --include="*.md, *.txt"
|
||||
|
||||
refute_pointer "refs/heads/master" "a.md"
|
||||
refute_pointer "refs/heads/master" "a.txt"
|
||||
|
||||
refute_local_object "$md_oid" "50"
|
||||
refute_local_object "$txt_oid" "30"
|
||||
)
|
||||
end_test
|
||||
|
||||
begin_test "migrate export (invalid --remote)"
|
||||
(
|
||||
set -e
|
||||
|
||||
setup_single_remote_branch_tracked
|
||||
|
||||
git lfs migrate export --include="*" --remote="zz" 2>&1 | tee migrate.log
|
||||
if [ ${PIPESTATUS[0]} -eq 0 ]; then
|
||||
echo >&2 "fatal: expected git lfs migrate export to fail, didn't"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
grep "fatal: invalid remote zz provided" migrate.log
|
||||
)
|
||||
end_test
|
||||
@ -84,6 +84,33 @@ setup_local_branch_with_nested_gitattrs() {
|
||||
git commit -m "add nested .gitattributes"
|
||||
}
|
||||
|
||||
# setup_single_local_branch_tracked creates a repository as follows:
|
||||
#
|
||||
# A---B
|
||||
# \
|
||||
# refs/heads/master
|
||||
#
|
||||
# - Commit 'A' has 120, in a.txt and 140 in a.md, with both files tracked as
|
||||
# pointers in Git LFS
|
||||
setup_single_local_branch_tracked() {
|
||||
set -e
|
||||
|
||||
reponame="migrate-single-remote-branch-with-attrs"
|
||||
|
||||
remove_and_create_local_repo "$reponame"
|
||||
|
||||
git lfs track "*.txt" "*.md"
|
||||
|
||||
git add .gitattributes
|
||||
git commit -m "initial commit"
|
||||
|
||||
base64 < /dev/urandom | head -c 120 > a.txt
|
||||
base64 < /dev/urandom | head -c 140 > a.md
|
||||
|
||||
git add a.txt a.md
|
||||
git commit -m "add a.{txt,md}"
|
||||
}
|
||||
|
||||
# setup_multiple_local_branches creates a repository as follows:
|
||||
#
|
||||
# B
|
||||
@ -134,6 +161,36 @@ setup_multiple_local_branches_with_gitattrs() {
|
||||
git commit -m "add .gitattributes"
|
||||
}
|
||||
|
||||
# setup_multiple_local_branches_tracked creates a repo with exactly the same
|
||||
# structure as in setup_multiple_local_branches, but with all files tracked by
|
||||
# Git LFS
|
||||
setup_multiple_local_branches_tracked() {
|
||||
set -e
|
||||
|
||||
reponame="migrate-info-multiple-local-branches"
|
||||
|
||||
remove_and_create_local_repo "$reponame"
|
||||
|
||||
git lfs track "*.txt" "*.md"
|
||||
git add .gitattributes
|
||||
git commit -m "initial commit"
|
||||
|
||||
base64 < /dev/urandom | head -c 120 > a.txt
|
||||
base64 < /dev/urandom | head -c 140 > a.md
|
||||
|
||||
git add a.txt a.md
|
||||
git commit -m "add a.{txt,md}"
|
||||
|
||||
git checkout -b my-feature
|
||||
|
||||
base64 < /dev/urandom | head -c 30 > a.md
|
||||
|
||||
git add a.md
|
||||
git commit -m "add an additional 30 bytes to a.md"
|
||||
|
||||
git checkout master
|
||||
}
|
||||
|
||||
# setup_local_branch_with_space creates a repository as follows:
|
||||
#
|
||||
# A
|
||||
@ -201,6 +258,34 @@ setup_single_remote_branch_with_gitattrs() {
|
||||
git commit -m "add .gitattributes"
|
||||
}
|
||||
|
||||
# Creates a repo identical to setup_single_remote_branch, except with *.md and
|
||||
# *.txt files tracked by Git LFS
|
||||
setup_single_remote_branch_tracked() {
|
||||
set -e
|
||||
|
||||
reponame="migrate-info-single-remote-branch"
|
||||
|
||||
remove_and_create_remote_repo "$reponame"
|
||||
|
||||
git lfs track "*.md" "*.txt"
|
||||
git add .gitattributes
|
||||
git commit -m "initial commit"
|
||||
|
||||
base64 < /dev/urandom | head -c 120 > a.txt
|
||||
base64 < /dev/urandom | head -c 140 > a.md
|
||||
|
||||
git add a.txt a.md
|
||||
git commit -m "add a.{txt,md}"
|
||||
|
||||
git push origin master
|
||||
|
||||
base64 < /dev/urandom | head -c 30 > a.txt
|
||||
base64 < /dev/urandom | head -c 50 > a.md
|
||||
|
||||
git add a.md a.txt
|
||||
git commit -m "add an additional 30, 50 bytes to a.{txt,md}"
|
||||
}
|
||||
|
||||
# setup_multiple_remote_branches creates a repository as follows:
|
||||
#
|
||||
# C
|
||||
@ -247,6 +332,41 @@ setup_multiple_remote_branches() {
|
||||
git checkout master
|
||||
}
|
||||
|
||||
# Creates a repo identical to that in setup_multiple_remote_branches(), but
|
||||
# with all files tracked by Git LFS
|
||||
setup_multiple_remote_branches_gitattrs() {
|
||||
set -e
|
||||
|
||||
reponame="migrate-info-exclude-remote-refs-given-branch"
|
||||
|
||||
remove_and_create_remote_repo "$reponame"
|
||||
|
||||
git lfs track "*.txt" "*.md"
|
||||
git add .gitattributes
|
||||
git commit -m "initial commit"
|
||||
|
||||
base64 < /dev/urandom | head -c 10 > a.txt
|
||||
base64 < /dev/urandom | head -c 11 > a.md
|
||||
git add a.txt a.md
|
||||
git commit -m "add 10, 11 bytes, a.{txt,md}"
|
||||
|
||||
git push origin master
|
||||
|
||||
base64 < /dev/urandom | head -c 20 > a.txt
|
||||
base64 < /dev/urandom | head -c 21 > a.md
|
||||
git add a.txt a.md
|
||||
git commit -m "add 20, 21 bytes, a.{txt,md}"
|
||||
|
||||
git checkout -b my-feature
|
||||
|
||||
base64 < /dev/urandom | head -c 30 > a.txt
|
||||
base64 < /dev/urandom | head -c 31 > a.md
|
||||
git add a.txt a.md
|
||||
git commit -m "add 30, 31 bytes, a.{txt,md}"
|
||||
|
||||
git checkout master
|
||||
}
|
||||
|
||||
# setup_single_local_branch_with_tags creates a repository as follows:
|
||||
#
|
||||
# A---B
|
||||
|
||||
@ -24,6 +24,31 @@ assert_pointer() {
|
||||
fi
|
||||
}
|
||||
|
||||
# refute_pointer confirms that the file in the repository for $path in the
|
||||
# given $ref is _not_ a pointer.
|
||||
#
|
||||
# $ refute_pointer "master" "path/to/file"
|
||||
refute_pointer() {
|
||||
local ref="$1"
|
||||
local path="$2"
|
||||
|
||||
gitblob=$(git ls-tree -lrz "$ref" |
|
||||
while read -r -d $'\0' x; do
|
||||
echo $x
|
||||
done |
|
||||
grep "$path" | cut -f 3 -d " ")
|
||||
|
||||
file=$(git cat-file -p $gitblob)
|
||||
version="version https://git-lfs.github.com/spec/v[0-9]"
|
||||
oid="oid sha256:[0-9a-f]\{32\}"
|
||||
size="size [0-9]*"
|
||||
regex="$version.*$oid.*$size"
|
||||
|
||||
if echo $file | grep -q "$regex"; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# assert_local_object confirms that an object file is stored for the given oid &
|
||||
# has the correct size
|
||||
# $ assert_local_object "some-oid" size
|
||||
|
||||
Loading…
Reference in New Issue
Block a user