2017-06-21 16:43:56 +00:00
|
|
|
package commands
|
|
|
|
|
2017-06-23 18:41:55 +00:00
|
|
|
import (
|
|
|
|
"bufio"
|
|
|
|
"bytes"
|
|
|
|
"encoding/hex"
|
|
|
|
"fmt"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"strings"
|
|
|
|
|
2018-05-23 16:36:36 +00:00
|
|
|
"github.com/git-lfs/git-lfs/errors"
|
2017-06-23 18:41:55 +00:00
|
|
|
"github.com/git-lfs/git-lfs/filepathfilter"
|
|
|
|
"github.com/git-lfs/git-lfs/git"
|
|
|
|
"github.com/git-lfs/git-lfs/git/githistory"
|
|
|
|
"github.com/git-lfs/git-lfs/git/odb"
|
2017-10-24 18:07:53 +00:00
|
|
|
"github.com/git-lfs/git-lfs/lfs"
|
2017-11-22 22:07:24 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tasklog"
|
2017-06-23 18:41:55 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tools"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
)
|
2017-06-21 16:43:56 +00:00
|
|
|
|
|
|
|
func migrateImportCommand(cmd *cobra.Command, args []string) {
|
2017-11-22 22:07:24 +00:00
|
|
|
l := tasklog.NewLogger(os.Stderr)
|
2017-09-22 22:12:20 +00:00
|
|
|
defer l.Close()
|
2017-08-29 21:05:11 +00:00
|
|
|
|
2017-06-23 18:41:55 +00:00
|
|
|
db, err := getObjectDatabase()
|
|
|
|
if err != nil {
|
|
|
|
ExitWithError(err)
|
|
|
|
}
|
2017-09-19 20:07:37 +00:00
|
|
|
defer db.Close()
|
|
|
|
|
2018-05-23 16:36:36 +00:00
|
|
|
if migrateNoRewrite {
|
2018-05-25 21:06:54 +00:00
|
|
|
if len(args) == 0 {
|
2018-05-30 16:54:45 +00:00
|
|
|
ExitWithError(errors.Errorf("fatal: expected one or more files with --no-rewrite"))
|
2018-05-25 21:06:54 +00:00
|
|
|
}
|
|
|
|
|
2018-05-23 16:36:36 +00:00
|
|
|
ref, err := git.CurrentRef()
|
|
|
|
if err != nil {
|
|
|
|
ExitWithError(errors.Wrap(err, "fatal: unable to find current reference"))
|
|
|
|
}
|
|
|
|
|
|
|
|
sha, _ := hex.DecodeString(ref.Sha)
|
|
|
|
commit, err := db.Commit(sha)
|
|
|
|
if err != nil {
|
|
|
|
ExitWithError(errors.Wrap(err, "fatal: unable to load commit"))
|
|
|
|
}
|
|
|
|
|
|
|
|
root := commit.TreeID
|
|
|
|
|
2018-05-25 21:06:54 +00:00
|
|
|
filter := git.GetAttributeFilter(cfg.LocalWorkingDir(), cfg.LocalGitDir())
|
2018-05-31 18:19:45 +00:00
|
|
|
if len(filter.Include()) == 0 {
|
2018-05-30 17:55:02 +00:00
|
|
|
ExitWithError(errors.Errorf("fatal: no Git LFS filters found in .gitattributes"))
|
2018-05-24 17:46:59 +00:00
|
|
|
}
|
|
|
|
|
2018-05-30 16:54:45 +00:00
|
|
|
gf := lfs.NewGitFilter(cfg)
|
|
|
|
|
2018-05-25 21:06:54 +00:00
|
|
|
for _, file := range args {
|
2018-05-30 17:18:04 +00:00
|
|
|
if !filter.Allows(file) {
|
2018-05-30 17:55:02 +00:00
|
|
|
ExitWithError(errors.Errorf("fatal: file %s did not match any Git LFS filters in .gitattributes", file))
|
2018-05-23 16:36:36 +00:00
|
|
|
}
|
2018-05-30 17:18:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, file := range args {
|
|
|
|
root, err = rewriteTree(gf, db, root, file)
|
|
|
|
if err != nil {
|
|
|
|
ExitWithError(errors.Wrapf(err, "fatal: could not rewrite %q", file))
|
|
|
|
}
|
2018-05-23 16:36:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
name, email := cfg.CurrentCommitter()
|
|
|
|
author := fmt.Sprintf("%s <%s>", name, email)
|
|
|
|
|
|
|
|
oid, err := db.WriteCommit(&odb.Commit{
|
|
|
|
Author: author,
|
|
|
|
Committer: author,
|
|
|
|
ParentIDs: [][]byte{sha},
|
2018-05-30 16:54:45 +00:00
|
|
|
Message: generateMigrateCommitMessage(cmd, strings.Join(args, ",")),
|
2018-05-23 16:36:36 +00:00
|
|
|
TreeID: root,
|
|
|
|
})
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
ExitWithError(errors.Wrap(err, "fatal: unable to write commit"))
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := git.UpdateRef(ref, oid, "git lfs migrate import --no-rewrite"); err != nil {
|
|
|
|
ExitWithError(errors.Wrap(err, "fatal: unable to update ref"))
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := checkoutNonBare(l); err != nil {
|
|
|
|
ExitWithError(errors.Wrap(err, "fatal: could not checkout"))
|
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-08-29 21:05:11 +00:00
|
|
|
rewriter := getHistoryRewriter(cmd, db, l)
|
2017-06-23 18:41:55 +00:00
|
|
|
|
|
|
|
tracked := trackedFromFilter(rewriter.Filter())
|
|
|
|
exts := tools.NewOrderedSet()
|
2017-10-24 18:07:53 +00:00
|
|
|
gitfilter := lfs.NewGitFilter(cfg)
|
2017-06-23 18:41:55 +00:00
|
|
|
|
2017-08-29 21:05:11 +00:00
|
|
|
migrate(args, rewriter, l, &githistory.RewriteOptions{
|
2018-03-19 13:41:35 +00:00
|
|
|
Verbose: migrateVerbose,
|
|
|
|
ObjectMapFilePath: objectMapFilePath,
|
2017-06-23 18:41:55 +00:00
|
|
|
BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) {
|
2017-06-26 17:27:21 +00:00
|
|
|
if filepath.Base(path) == ".gitattributes" {
|
|
|
|
return b, nil
|
|
|
|
}
|
|
|
|
|
2017-06-23 18:41:55 +00:00
|
|
|
var buf bytes.Buffer
|
|
|
|
|
2017-10-24 18:07:53 +00:00
|
|
|
if _, err := clean(gitfilter, &buf, b.Contents, path, b.Size); err != nil {
|
2017-06-23 18:41:55 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-06-26 15:07:18 +00:00
|
|
|
if ext := filepath.Ext(path); len(ext) > 0 {
|
2017-06-23 18:41:55 +00:00
|
|
|
exts.Add(fmt.Sprintf("*%s filter=lfs diff=lfs merge=lfs -text", ext))
|
|
|
|
}
|
|
|
|
|
|
|
|
return &odb.Blob{
|
|
|
|
Contents: &buf, Size: int64(buf.Len()),
|
|
|
|
}, nil
|
|
|
|
},
|
|
|
|
|
|
|
|
TreeCallbackFn: func(path string, t *odb.Tree) (*odb.Tree, error) {
|
2018-02-19 20:49:05 +00:00
|
|
|
if path != "/" {
|
2017-06-23 18:41:55 +00:00
|
|
|
// Ignore non-root trees.
|
|
|
|
return t, nil
|
|
|
|
}
|
|
|
|
|
2017-06-26 16:51:51 +00:00
|
|
|
ours := tracked
|
|
|
|
if ours.Cardinality() == 0 {
|
2017-06-23 18:41:55 +00:00
|
|
|
// If there were no explicitly tracked
|
|
|
|
// --include, --exclude filters, assume that the
|
|
|
|
// include set is the wildcard filepath
|
|
|
|
// extensions of files tracked.
|
2017-06-26 16:51:51 +00:00
|
|
|
ours = exts
|
2018-06-08 20:15:48 +00:00
|
|
|
|
|
|
|
if ours.Cardinality() == 0 {
|
|
|
|
// If it is still the case that we have
|
|
|
|
// no patterns to track, that means that
|
|
|
|
// we are in a tree that does not
|
|
|
|
// require .gitattributes changes.
|
|
|
|
//
|
|
|
|
// We can return early to avoid
|
|
|
|
// comparing and saving an identical
|
|
|
|
// tree.
|
|
|
|
return t, nil
|
|
|
|
}
|
2017-06-23 18:41:55 +00:00
|
|
|
}
|
|
|
|
|
2017-06-26 16:51:51 +00:00
|
|
|
theirs, err := trackedFromAttrs(db, t)
|
2017-06-23 18:41:55 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create a blob of the attributes that are optionally
|
|
|
|
// present in the "t" tree's .gitattributes blob, and
|
|
|
|
// union in the patterns that we've tracked.
|
|
|
|
//
|
|
|
|
// Perform this Union() operation each time we visit a
|
|
|
|
// root tree such that if the underlying .gitattributes
|
|
|
|
// is present and has a diff between commits in the
|
|
|
|
// range of commits to migrate, those changes are
|
|
|
|
// preserved.
|
2017-06-26 16:51:51 +00:00
|
|
|
blob, err := trackedToBlob(db, theirs.Clone().Union(ours))
|
2017-06-23 18:41:55 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Finally, return a copy of the tree "t" that has the
|
|
|
|
// new .gitattributes file included/replaced.
|
|
|
|
return t.Merge(&odb.TreeEntry{
|
|
|
|
Name: ".gitattributes",
|
|
|
|
Filemode: 0100644,
|
|
|
|
Oid: blob,
|
|
|
|
}), nil
|
|
|
|
},
|
|
|
|
|
|
|
|
UpdateRefs: true,
|
|
|
|
})
|
|
|
|
|
2018-05-23 16:36:36 +00:00
|
|
|
if err := checkoutNonBare(l); err != nil {
|
|
|
|
ExitWithError(errors.Wrap(err, "fatal: could not checkout"))
|
|
|
|
}
|
|
|
|
}
|
2017-09-22 21:57:07 +00:00
|
|
|
|
2018-05-23 16:36:36 +00:00
|
|
|
// generateMigrateCommitMessage generates a commit message used with
|
|
|
|
// --no-rewrite, using --message (if given) or generating one if it isn't.
|
2018-05-30 16:54:45 +00:00
|
|
|
func generateMigrateCommitMessage(cmd *cobra.Command, patterns string) string {
|
2018-05-23 16:36:36 +00:00
|
|
|
if cmd.Flag("message").Changed {
|
|
|
|
return migrateCommitMessage
|
2017-06-23 18:41:55 +00:00
|
|
|
}
|
2018-05-30 16:54:45 +00:00
|
|
|
return fmt.Sprintf("%s: convert to Git LFS", patterns)
|
2018-05-23 16:36:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// checkoutNonBare forces a checkout of the current reference, so long as the
|
|
|
|
// repository is non-bare.
|
|
|
|
//
|
|
|
|
// It returns nil on success, and a non-nil error on failure.
|
|
|
|
func checkoutNonBare(l *tasklog.Logger) error {
|
|
|
|
if bare, _ := git.IsBare(); bare {
|
|
|
|
return nil
|
2017-06-23 18:41:55 +00:00
|
|
|
}
|
2018-05-23 16:36:36 +00:00
|
|
|
|
|
|
|
t := l.Waiter("migrate: checkout")
|
|
|
|
defer t.Complete()
|
|
|
|
|
|
|
|
return git.Checkout("", nil, true)
|
2017-06-23 18:41:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// trackedFromFilter returns an ordered set of strings where each entry is a
|
|
|
|
// line in the .gitattributes file. It adds/removes the fiter/diff/merge=lfs
|
|
|
|
// attributes based on patterns included/excldued in the given filter.
|
|
|
|
func trackedFromFilter(filter *filepathfilter.Filter) *tools.OrderedSet {
|
|
|
|
tracked := tools.NewOrderedSet()
|
|
|
|
|
|
|
|
for _, include := range filter.Include() {
|
2018-03-21 00:36:04 +00:00
|
|
|
tracked.Add(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text", escapeAttrPattern(include)))
|
2017-06-23 18:41:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, exclude := range filter.Exclude() {
|
2018-03-21 00:36:04 +00:00
|
|
|
tracked.Add(fmt.Sprintf("%s text -filter -merge -diff", escapeAttrPattern(exclude)))
|
2017-06-23 18:41:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return tracked
|
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
|
|
|
// attrsCache maintains a cache from the hex-encoded SHA1 of a
|
|
|
|
// .gitattributes blob to the set of patterns parsed from that blob.
|
2017-06-26 17:27:21 +00:00
|
|
|
attrsCache = make(map[string]*tools.OrderedSet)
|
2017-06-23 18:41:55 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// trackedFromAttrs returns an ordered line-delimited set of the contents of a
|
|
|
|
// .gitattributes blob in a given tree "t".
|
|
|
|
//
|
|
|
|
// It returns an empty set if no attributes file could be found, or an error if
|
|
|
|
// it could not otherwise be opened.
|
|
|
|
func trackedFromAttrs(db *odb.ObjectDatabase, t *odb.Tree) (*tools.OrderedSet, error) {
|
|
|
|
var oid []byte
|
|
|
|
|
|
|
|
for _, e := range t.Entries {
|
|
|
|
if strings.ToLower(e.Name) == ".gitattributes" && e.Type() == odb.BlobObjectType {
|
|
|
|
oid = e.Oid
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if oid == nil {
|
|
|
|
// TODO(@ttaylorr): make (*tools.OrderedSet)(nil) a valid
|
|
|
|
// receiver for non-mutative methods.
|
|
|
|
return tools.NewOrderedSet(), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
sha1 := hex.EncodeToString(oid)
|
|
|
|
|
|
|
|
if s, ok := attrsCache[sha1]; ok {
|
|
|
|
return s, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
blob, err := db.Blob(oid)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
attrs := tools.NewOrderedSet()
|
|
|
|
|
|
|
|
scanner := bufio.NewScanner(blob.Contents)
|
|
|
|
for scanner.Scan() {
|
|
|
|
attrs.Add(scanner.Text())
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := scanner.Err(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
attrsCache[sha1] = attrs
|
|
|
|
|
|
|
|
return attrsCache[sha1], nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// trackedToBlob writes and returns the OID of a .gitattributes blob based on
|
|
|
|
// the patterns given in the ordered set of patterns, "patterns".
|
|
|
|
func trackedToBlob(db *odb.ObjectDatabase, patterns *tools.OrderedSet) ([]byte, error) {
|
|
|
|
var attrs bytes.Buffer
|
|
|
|
|
|
|
|
for pattern := range patterns.Iter() {
|
|
|
|
fmt.Fprintf(&attrs, "%s\n", pattern)
|
|
|
|
}
|
2017-06-21 16:43:56 +00:00
|
|
|
|
2017-06-23 18:41:55 +00:00
|
|
|
return db.WriteBlob(&odb.Blob{
|
|
|
|
Contents: &attrs,
|
|
|
|
Size: int64(attrs.Len()),
|
|
|
|
})
|
2017-06-21 16:43:56 +00:00
|
|
|
}
|
2018-05-23 16:36:36 +00:00
|
|
|
|
2018-05-24 17:46:59 +00:00
|
|
|
// rewriteTree replaces the blob at the provided path within the given tree with
|
|
|
|
// a git lfs pointer. It will recursively rewrite any subtrees along the path to the
|
|
|
|
// blob.
|
2018-05-30 16:54:45 +00:00
|
|
|
func rewriteTree(gf *lfs.GitFilter, db *odb.ObjectDatabase, root []byte, path string) ([]byte, error) {
|
2018-05-24 17:46:59 +00:00
|
|
|
tree, err := db.Tree(root)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
splits := strings.SplitN(path, "/", 2)
|
|
|
|
|
|
|
|
switch len(splits) {
|
|
|
|
case 1:
|
|
|
|
// The path points to an entry at the root of this tree, so it must be a blob.
|
2018-05-30 16:54:45 +00:00
|
|
|
// Try to replace this blob with a Git LFS pointer.
|
2018-05-24 17:46:59 +00:00
|
|
|
index := findEntry(tree, splits[0])
|
|
|
|
if index < 0 {
|
|
|
|
return nil, errors.Errorf("unable to find entry %s in tree", splits[0])
|
|
|
|
}
|
|
|
|
|
|
|
|
blobEntry := tree.Entries[index]
|
|
|
|
blob, err := db.Blob(blobEntry.Oid)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
var buf bytes.Buffer
|
|
|
|
|
2018-05-30 16:54:45 +00:00
|
|
|
if _, err := clean(gf, &buf, blob.Contents, blobEntry.Name, blob.Size); err != nil {
|
2018-05-24 17:46:59 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2018-05-30 16:54:45 +00:00
|
|
|
newOid, err := db.WriteBlob(&odb.Blob{
|
2018-05-24 17:46:59 +00:00
|
|
|
Contents: &buf,
|
|
|
|
Size: int64(buf.Len()),
|
2018-05-30 16:54:45 +00:00
|
|
|
})
|
2018-05-24 17:46:59 +00:00
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2018-05-30 16:54:45 +00:00
|
|
|
tree = tree.Merge(&odb.TreeEntry{
|
2018-05-24 17:46:59 +00:00
|
|
|
Name: splits[0],
|
|
|
|
Filemode: blobEntry.Filemode,
|
|
|
|
Oid: newOid,
|
2018-05-30 16:54:45 +00:00
|
|
|
})
|
2018-05-24 17:46:59 +00:00
|
|
|
return db.WriteTree(tree)
|
|
|
|
|
|
|
|
case 2:
|
|
|
|
// The path points to an entry in a subtree contained at the root of the tree.
|
|
|
|
// Recursively rewrite the subtree.
|
|
|
|
head, tail := splits[0], splits[1]
|
|
|
|
|
|
|
|
index := findEntry(tree, head)
|
|
|
|
if index < 0 {
|
|
|
|
return nil, errors.Errorf("unable to find entry %s in tree", head)
|
|
|
|
}
|
|
|
|
|
|
|
|
subtreeEntry := tree.Entries[index]
|
|
|
|
if subtreeEntry.Type() != odb.TreeObjectType {
|
|
|
|
return nil, errors.Errorf("migrate: expected %s to be a tree, got %s", head, subtreeEntry.Type())
|
|
|
|
}
|
|
|
|
|
2018-05-30 16:54:45 +00:00
|
|
|
rewrittenSubtree, err := rewriteTree(gf, db, subtreeEntry.Oid, tail)
|
2018-05-24 17:46:59 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
tree = tree.Merge(&odb.TreeEntry{
|
|
|
|
Filemode: subtreeEntry.Filemode,
|
|
|
|
Name: subtreeEntry.Name,
|
|
|
|
Oid: rewrittenSubtree,
|
|
|
|
})
|
|
|
|
|
|
|
|
return db.WriteTree(tree)
|
|
|
|
|
|
|
|
default:
|
|
|
|
return nil, errors.Errorf("error parsing path %s", path)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-05-30 16:54:45 +00:00
|
|
|
// findEntry searches a tree for the desired entry, and returns the index of that
|
|
|
|
// entry within the tree's Entries array
|
2018-05-24 17:46:59 +00:00
|
|
|
func findEntry(t *odb.Tree, name string) int {
|
|
|
|
for i, entry := range t.Entries {
|
|
|
|
if entry.Name == name {
|
|
|
|
return i
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return -1
|
|
|
|
}
|