rework 'push --all' so it's consistent with fetch

commands/command_push.go

@@ -4,7 +4,6 @@ import (
 	"io/ioutil"
 	"os"
 
-	"github.com/github/git-lfs/git"
 	"github.com/github/git-lfs/lfs"
 	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
 	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
@@ -19,7 +18,7 @@ var (
 	pushDryRun       = false
 	pushDeleteBranch = "(delete)"
 	pushObjectIDs    = false
-	pushAllToRemote  = false
+	pushAll          = false
 	useStdin         = false
 
 	// shares some global vars and functions with commmands_pre_push.go
@@ -33,24 +32,51 @@ func uploadsBetweenRefs(left string, right string) *lfs.TransferQueue {
 	if err != nil {
 		Panic(err, "Error scanning for Git LFS files")
 	}
-	return uploadPointers(pointers)
+	return uploadPointers(pointers, nil)
 }
 
-func uploadsBetweenRefAndRemote(ref, remote string) *lfs.TransferQueue {
+func uploadsBetweenRefAndRemote(remote string, refs []string) *lfs.TransferQueue {
-	tracerx.Printf("Upload between %v and remote %v", ref, remote)
+	tracerx.Printf("Upload refs %v to remote %v", remote, refs)
 
 	scanOpt := &lfs.ScanRefsOptions{ScanMode: lfs.ScanLeftToRemoteMode, RemoteName: remote}
-	if pushAllToRemote {
+	if pushAll {
-		scanOpt.ScanMode = lfs.ScanRefsMode
+		if len(refs) == 0 {
+			// no ref given as an arg, so scan all refs
+			pointers, err := lfs.ScanRefs("", "", scanOpt)
+			if err != nil {
+				Panic(err, "Error scanning for all Git LFS files")
+			}
+			return uploadPointers(pointers, nil)
+		} else {
+			scanOpt.ScanMode = lfs.ScanRefsMode
+		}
 	}
-	pointers, err := lfs.ScanRefs(ref, "", scanOpt)
+
-	if err != nil {
+	// keep a unique set of pointers
-		Panic(err, "Error scanning for Git LFS files")
+	oidPointerMap := make(map[string]*lfs.WrappedPointer)
+
+	for _, ref := range refs {
+		pointers, err := lfs.ScanRefs(ref, "", scanOpt)
+		if err != nil {
+			Panic(err, "Error scanning for Git LFS files in the %q ref", ref)
+		}
+
+		for _, p := range pointers {
+			oidPointerMap[p.Oid] = p
+		}
 	}
-	return uploadPointers(pointers)
+
+	i := 0
+	pointers := make([]*lfs.WrappedPointer, len(oidPointerMap))
+	for _, pointer := range oidPointerMap {
+		pointers[i] = pointer
+		i += 1
+	}
+
+	return uploadPointers(pointers, nil)
 }
 
-func uploadPointers(pointers []*lfs.WrappedPointer) *lfs.TransferQueue {
+func uploadPointers(pointers []*lfs.WrappedPointer, skipObjects map[string]struct{}) *lfs.TransferQueue {
 	totalSize := int64(0)
 	for _, p := range pointers {
 		totalSize += p.Size
@@ -159,27 +185,12 @@ func pushCommand(cmd *cobra.Command, args []string) {
 
 		uploadQueue = uploadsWithObjectIDs(args[1:])
 	} else {
-
 		if len(args) < 1 {
 			Print("Usage: git lfs push --dry-run <remote> [ref]")
 			return
 		}
 
-		remote := args[0]
+		uploadQueue = uploadsBetweenRefAndRemote(args[0], args[1:])
-		var ref string
-		if len(args) == 2 {
-			ref = args[1]
-		}
-
-		if ref == "" {
-			localRef, err := git.CurrentRef()
-			if err != nil {
-				Panic(err, "Error getting local ref")
-			}
-			ref = localRef.Sha
-		}
-
-		uploadQueue = uploadsBetweenRefAndRemote(ref, remote)
 	}
 
 	if !pushDryRun {
@@ -202,7 +213,7 @@ func init() {
 	pushCmd.Flags().BoolVarP(&pushDryRun, "dry-run", "d", false, "Do everything except actually send the updates")
 	pushCmd.Flags().BoolVarP(&useStdin, "stdin", "s", false, "Take refs on stdin (for pre-push hook)")
 	pushCmd.Flags().BoolVarP(&pushObjectIDs, "object-id", "o", false, "Push LFS object ID(s)")
-	pushCmd.Flags().BoolVarP(&pushAllToRemote, "all", "a", false, "Push all objects for the current ref to the remote.")
+	pushCmd.Flags().BoolVarP(&pushAll, "all", "a", false, "Push all objects for the current ref to the remote.")
 
 	RootCmd.AddCommand(pushCmd)
 }

test/test-push.sh

@@ -15,6 +15,10 @@ begin_test "push"
   git add .gitattributes a.dat
   git commit -m "add a.dat"
 
+  git lfs push --dry-run origin master 2>&1 | tee push.log
+  grep "push 4c48d2a6991c9895bcddcf027e1e4907280bcf21975492b1afbade396d6a3340 => a.dat" push.log
+  [ $(wc -l < push.log) -eq 1 ]
+
   git lfs push origin master 2>&1 | tee push.log
   grep "(1 of 1 files)" push.log
 
@@ -23,11 +27,178 @@ begin_test "push"
   git add b.dat
   git commit -m "add b.dat"
 
+  git lfs push --dry-run origin push-b 2>&1 | tee push.log
+  grep "push 4c48d2a6991c9895bcddcf027e1e4907280bcf21975492b1afbade396d6a3340 => a.dat" push.log
+  grep "push 82be50ad35070a4ef3467a0a650c52d5b637035e7ad02c36652e59d01ba282b7 => b.dat" push.log
+  [ $(wc -l < push.log) -eq 2 ]
+
   git lfs push origin push-b 2>&1 | tee push.log
   grep "(1 of 2 files, 1 skipped)" push.log
 )
 end_test
 
+# sets up the tests for the next few push --all tests
+push_all_setup() {
+  local suffix="$1"
+  reponame="$(basename "$0" ".sh")-all"
+  setup_remote_repo "$reponame"
+  clone_repo "$reponame" "repo-all-$suffix"
+
+  git lfs track "*.dat"
+  content1="initial"
+  content2="update"
+  content3="branch"
+  content4="tagged"
+  content5="master"
+  extracontent="extra"
+  oid1=$(printf "$content1" | shasum -a 256 | cut -f 1 -d " ")
+  oid2=$(printf "$content2" | shasum -a 256 | cut -f 1 -d " ")
+  oid3=$(printf "$content3" | shasum -a 256 | cut -f 1 -d " ")
+  oid4=$(printf "$content4" | shasum -a 256 | cut -f 1 -d " ")
+  oid5=$(printf "$content5" | shasum -a 256 | cut -f 1 -d " ")
+  extraoid=$(printf "$extracontent" | shasum -a 256 | cut -f 1 -d " ")
+
+  echo "[
+  {
+    \"CommitDate\":\"$(get_date -6m)\",
+    \"Files\":[
+      {\"Filename\":\"file1.dat\",\"Size\":${#content1},\"Data\":\"$content1\"}
+    ]
+  },
+  {
+    \"CommitDate\":\"$(get_date -5m)\",
+    \"Files\":[
+      {\"Filename\":\"file1.dat\",\"Size\":${#content2},\"Data\":\"$content2\"}
+    ]
+  },
+  {
+    \"CommitDate\":\"$(get_date -4m)\",
+    \"NewBranch\":\"branch\",
+    \"Files\":[
+      {\"Filename\":\"file1.dat\",\"Size\":${#content3},\"Data\":\"$content3\"}
+    ]
+  },
+  {
+    \"CommitDate\":\"$(get_date -4m)\",
+    \"ParentBranches\":[\"master\"],
+    \"Tags\":[\"tag\"],
+    \"Files\":[
+      {\"Filename\":\"file1.dat\",\"Size\":${#content4},\"Data\":\"$content4\"}
+    ]
+  },
+  {
+    \"CommitDate\":\"$(get_date -2m)\",
+    \"Files\":[
+      {\"Filename\":\"file1.dat\",\"Size\":${#content5},\"Data\":\"$content5\"},
+      {\"Filename\":\"file2.dat\",\"Size\":${#extracontent},\"Data\":\"$extracontent\"}
+    ]
+  }
+  ]" | lfstest-testutils addcommits
+
+  git rm file2.dat
+  git commit -m "remove file2.dat"
+
+  setup_alternate_remote "$reponame-$suffix"
+}
+
+begin_test "push --all (no ref args)"
+(
+  set -e
+
+  push_all_setup "everything"
+
+  git lfs push --dry-run --all origin 2>&1 | tee push.log
+  grep "push $oid1 => file1.dat" push.log
+  grep "push $oid2 => file1.dat" push.log
+  grep "push $oid3 => file1.dat" push.log
+  grep "push $oid4 => file1.dat" push.log
+  grep "push $oid5 => file1.dat" push.log
+  grep "push $extraoid => file2.dat" push.log
+  [ $(wc -l < push.log) -eq 6 ]
+
+  git push --all origin 2>&1 | tee push.log
+  grep "5 files" push.log # should be 6?
+  assert_server_object "$reponame-everything" "$oid1"
+  assert_server_object "$reponame-everything" "$oid2"
+  assert_server_object "$reponame-everything" "$oid3"
+  assert_server_object "$reponame-everything" "$oid4"
+  assert_server_object "$reponame-everything" "$oid5"
+  assert_server_object "$reponame-everything" "$extraoid"
+)
+end_test
+
+begin_test "push --all (1 ref arg)"
+(
+  set -e
+
+  push_all_setup "ref"
+
+  git lfs push --dry-run --all origin branch 2>&1 | tee push.log
+  grep "push $oid1 => file1.dat" push.log
+  grep "push $oid2 => file1.dat" push.log
+  grep "push $oid3 => file1.dat" push.log
+  [ $(wc -l < push.log) -eq 3 ]
+
+  git lfs push --all origin branch 2>&1 | tee push.log
+  grep "3 files" push.log
+  assert_server_object "$reponame-ref" "$oid1"
+  assert_server_object "$reponame-ref" "$oid2"
+  assert_server_object "$reponame-ref" "$oid3"
+  refute_server_object "$reponame-ref" "$oid4"     # in master and the tag
+  refute_server_object "$reponame-ref" "$oid5"
+  refute_server_object "$reponame-ref" "$extraoid"
+)
+end_test
+
+begin_test "push --all (multiple ref args)"
+(
+  set -e
+
+  push_all_setup "multiple-refs"
+
+  git lfs push --dry-run --all origin branch tag 2>&1 | tee push.log
+  grep "push $oid1 => file1.dat" push.log
+  grep "push $oid2 => file1.dat" push.log
+  grep "push $oid3 => file1.dat" push.log
+  grep "push $oid4 => file1.dat" push.log
+  [ $(wc -l < push.log) -eq 4 ]
+
+  git lfs push --all origin branch tag 2>&1 | tee push.log
+  grep "4 files" push.log
+  assert_server_object "$reponame-multiple-refs" "$oid1"
+  assert_server_object "$reponame-multiple-refs" "$oid2"
+  assert_server_object "$reponame-multiple-refs" "$oid3"
+  assert_server_object "$reponame-multiple-refs" "$oid4"
+  refute_server_object "$reponame-multiple-refs" "$oid5"     # only in master
+  refute_server_object "$reponame-multiple-refs" "$extraoid"
+)
+end_test
+
+begin_test "push --all (ref with deleted files)"
+(
+  set -e
+
+  push_all_setup "ref-with-deleted"
+
+  git lfs push --dry-run --all origin master 2>&1 | tee push.log
+  grep "push $oid1 => file1.dat" push.log
+  grep "push $oid2 => file1.dat" push.log
+  grep "push $oid4 => file1.dat" push.log
+  grep "push $oid5 => file1.dat" push.log
+  grep "push $extraoid => file2.dat" push.log
+  [ $(wc -l < push.log) -eq 5 ]
+
+  git lfs push --all origin master 2>&1 | tee push.log
+  grep "5 files" push.log
+  assert_server_object "$reponame-ref-with-deleted" "$oid1"
+  assert_server_object "$reponame-ref-with-deleted" "$oid2"
+  refute_server_object "$reponame-ref-with-deleted" "$oid3" # only in the branch
+  assert_server_object "$reponame-ref-with-deleted" "$oid4"
+  assert_server_object "$reponame-ref-with-deleted" "$oid5"
+  assert_server_object "$reponame-ref-with-deleted" "$extraoid"
+)
+end_test
+
 begin_test "push dry-run"
 (
   set -e

test/testhelpers.sh

@@ -130,6 +130,24 @@ setup_remote_repo() {
   git config receive.denyCurrentBranch ignore
 }
 
+# creates a bare remote repository for a local clone. Useful to test pushing to
+# a fresh remote server.
+#
+#   $ setup_alternate_remote "$reponame-whatever"
+#   $ setup_alternate_remote "$reponame-whatever" "other-remote-name"
+#
+setup_alternate_remote() {
+  local newRemoteName=$1
+  local remote=${2:-origin}
+
+  wd=`pwd`
+
+  setup_remote_repo "$newRemoteName"
+  cd $wd
+  git remote rm "$remote"
+  git remote add "$remote" "$GITSERVER/$newRemoteName"
+}
+
 # clone_repo clones a repository from the test Git server to the subdirectory
 # $dir under $TRASHDIR. setup_remote_repo() needs to be run first.
 clone_repo() {
@@ -341,4 +359,4 @@ get_date() {
     done
     date $ARGS -u +%Y-%m-%dT%TZ
   fi
-}
+}