From 5069ecdfce3ad8fdfefc156ad5c2483960512cf8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtniemi Date: Sun, 31 Jan 2016 23:06:40 +0200 Subject: [PATCH] Add support for reference clone --- commands/command_fetch.go | 9 ++++ lfs/lfs.go | 25 ++++++++++ lfs/pointer_smudge.go | 11 ++++ lfs/util.go | 32 ++++++++++++ test/test-env.sh | 14 ++++++ test/test-reference-clone.sh | 97 ++++++++++++++++++++++++++++++++++++ test/test-worktree.sh | 2 + 7 files changed, 190 insertions(+) create mode 100755 test/test-reference-clone.sh diff --git a/commands/command_fetch.go b/commands/command_fetch.go index 3fe56c28..d87e9b78 100644 --- a/commands/command_fetch.go +++ b/commands/command_fetch.go @@ -282,6 +282,15 @@ func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []str // This avoids previous case of over-reporting a requirement for files we already have // which would only be skipped by PointerSmudgeObject later passFilter := lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude) + + if !lfs.ObjectExistsOfSize(p.Oid, p.Size) { + altMediafile := lfs.LocalReferencePath(p.Oid) + mediafile, err := lfs.LocalMediaPath(p.Oid) + if err == nil && altMediafile != "" && lfs.FileExistsOfSize(altMediafile, p.Size) { + lfs.LinkOrCopy(altMediafile, mediafile) + } + } + if !lfs.ObjectExistsOfSize(p.Oid, p.Size) && passFilter { tracerx.Printf("fetch %v [%v]", p.Name, p.Oid) q.Add(lfs.NewDownloadable(p)) diff --git a/lfs/lfs.go b/lfs/lfs.go index ebf6884c..1dd6aab9 100644 --- a/lfs/lfs.go +++ b/lfs/lfs.go @@ -31,6 +31,7 @@ var ( LocalGitStorageDir string // parent of objects/lfs (may be same as LocalGitDir but may not) LocalMediaDir string // root of lfs objects LocalObjectTempDir string // where temporarily downloading objects are stored + LocalReferenceDir string // alternative local media dir (relative to clone reference repo) objects *localstorage.LocalStorage LocalLogDir string checkedTempDir string @@ -56,6 +57,13 @@ func LocalMediaPath(oid string) (string, error) { return objects.BuildObjectPath(oid) } +func LocalReferencePath(sha string) string { + if LocalReferenceDir == "" { + return "" + } + return filepath.Join(LocalReferenceDir, sha[0:2], sha[2:4], sha) +} + func ObjectExistsOfSize(oid string, size int64) bool { path := objects.ObjectPath(oid) return FileExistsOfSize(path, size) @@ -69,6 +77,7 @@ func Environ() []string { fmt.Sprintf("LocalGitDir=%s", LocalGitDir), fmt.Sprintf("LocalGitStorageDir=%s", LocalGitStorageDir), fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir), + fmt.Sprintf("LocalReferenceDir=%s", LocalReferenceDir), fmt.Sprintf("TempDir=%s", TempDir), fmt.Sprintf("ConcurrentTransfers=%d", Config.ConcurrentTransfers()), fmt.Sprintf("BatchTransfer=%v", Config.BatchTransfer()), @@ -98,6 +107,7 @@ func ResolveDirs() { LocalWorkingDir = ResolveSymlinks(LocalWorkingDir) LocalGitStorageDir = resolveGitStorageDir(LocalGitDir) + LocalReferenceDir = resolveReferenceDir(LocalGitStorageDir) TempDir = filepath.Join(LocalGitDir, "lfs", "tmp") // temp files per worktree objs, err := localstorage.New( @@ -182,6 +192,21 @@ func processGitRedirectFile(file, prefix string) (string, error) { return dir, nil } +func resolveReferenceDir(gitStorageDir string) string { + cloneReferencePath := filepath.Join(gitStorageDir, "objects", "info", "alternates") + if FileExists(cloneReferencePath) { + buffer, err := ioutil.ReadFile(cloneReferencePath) + if err == nil { + path := strings.TrimSpace(string(buffer[:])) + referenceLfsStoragePath := filepath.Join(filepath.Dir(path), "lfs", "objects") + if DirExists(referenceLfsStoragePath) { + return referenceLfsStoragePath + } + } + } + return "" +} + // From a git dir, get the location that objects are to be stored (we will store lfs alongside) // Sometimes there is an additional level of redirect on the .git folder by way of a commondir file // before you find object storage, e.g. 'git worktree' uses this. It redirects to gitdir either by GIT_DIR diff --git a/lfs/pointer_smudge.go b/lfs/pointer_smudge.go index 36a95376..815f9d53 100644 --- a/lfs/pointer_smudge.go +++ b/lfs/pointer_smudge.go @@ -41,6 +41,17 @@ func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download } stat, statErr := os.Stat(mediafile) + + if statErr != nil || stat == nil { + altMediafile := LocalReferencePath(ptr.Oid) + if altMediafile != "" && FileExistsOfSize(altMediafile, ptr.Size) { + copyErr := LinkOrCopy(altMediafile, mediafile) + if copyErr == nil { + stat, statErr = os.Stat(mediafile) + } + } + } + if statErr == nil && stat != nil { fileSize := stat.Size() if fileSize == 0 || fileSize != ptr.Size { diff --git a/lfs/util.go b/lfs/util.go index 87fb4dc1..b202256d 100644 --- a/lfs/util.go +++ b/lfs/util.go @@ -326,3 +326,35 @@ func FileExistsOfSize(path string, sz int64) bool { return !fi.IsDir() && fi.Size() == sz } + +func CopyFileContents(src string, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + defer func() { + cerr := out.Close() + if err == nil { + err = cerr + } + }() + _, err = io.Copy(out, in) + if err != nil { + return err + } + err = out.Sync() + return err +} + +func LinkOrCopy(src string, dst string) error { + err := os.Link(src, dst) + if err == nil { + return err + } + return CopyFileContents(src, dst) +} diff --git a/test/test-env.sh b/test/test-env.sh index 38de4327..b2e1058b 100755 --- a/test/test-env.sh +++ b/test/test-env.sh @@ -27,6 +27,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -63,6 +64,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -106,6 +108,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -147,6 +150,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -190,6 +194,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -234,6 +239,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -280,6 +286,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=5 BatchTransfer=false @@ -332,6 +339,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -378,6 +386,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -417,6 +426,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -447,6 +457,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -466,6 +477,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -484,6 +496,7 @@ LocalWorkingDir=%s LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true @@ -513,6 +526,7 @@ LocalWorkingDir= LocalGitDir=%s LocalGitStorageDir=%s LocalMediaDir=%s +LocalReferenceDir= TempDir=%s ConcurrentTransfers=3 BatchTransfer=true diff --git a/test/test-reference-clone.sh b/test/test-reference-clone.sh new file mode 100755 index 00000000..92f82fd8 --- /dev/null +++ b/test/test-reference-clone.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +. "test/testlib.sh" + +assert_same_inode() { + local repo1=$1 + local repo2=$2 + local oid=$3 + + if ! uname -s | grep -qE 'CYGWIN|MSYS|MINGW'; then + cfg1=$(cd "$repo1"; git lfs env | grep LocalMediaDir) + f1="${cfg1:14}/${oid:0:2}/${oid:2:2}/$oid" + inode1=$(ls -i $f1 | cut -f1 -d\ ) + + cfg2=$(cd "$repo2"; git lfs env | grep LocalMediaDir) + f2="${cfg2:14}/${oid:0:2}/${oid:2:2}/$oid" + inode2=$(ls -i $f2 | cut -f1 -d\ ) + + [ "$inode1" == "$inode2" ] + fi +} + +begin_test "clone with reference" +( + set -e + + reponame="$(basename "$0" ".sh")" + setup_remote_repo "$reponame" + + ref_repo=clone_reference_repo + ref_repo_dir=$TRASHDIR/$ref_repo + clone_repo "$reponame" "$ref_repo" + git lfs track "*.dat" + contents="a" + oid=$(calc_oid "$contents") + + printf "$contents" > a.dat + git add a.dat + git add .gitattributes + git commit -m "add a.dat" 2>&1 + git push origin master + + delete_server_object "$reponame" "$oid" + + repo=test_repo + repo_dir=$TRASHDIR/$repo + git clone --reference "$ref_repo_dir/.git" \ + "$GITSERVER/$reponame" "$repo_dir" 2> clone.log + + if grep -q "Downloading a.dat" clone.log; then + exit 1 + fi + + cd "$TRASHDIR/$repo" + + assert_pointer "master" "a.dat" "$oid" 1 + assert_same_inode "$repo_dir" "$ref_repo_dir" "$oid" +) +end_test + +begin_test "fetch from clone reference" +( + set -e + + reponame="$(basename "$0" ".sh")2" + setup_remote_repo "$reponame" + + ref_repo=clone_reference_repo2 + ref_repo_dir=$TRASHDIR/$ref_repo + clone_repo "$reponame" "$ref_repo" + + repo=test_repo2 + repo_dir=$TRASHDIR/$repo + git clone --reference "$ref_repo_dir/.git" \ + "$GITSERVER/$reponame" "$repo_dir" 2> clone.log + + cd "$ref_repo_dir" + git lfs track "*.dat" + contents="a" + oid=$(calc_oid "$contents") + + printf "$contents" > a.dat + git add a.dat + git add .gitattributes + git commit -m "add a.dat" 2>&1 + git push origin master + + delete_server_object "$reponame" "$oid" + + cd "$repo_dir" + GIT_LFS_SKIP_SMUDGE=1 git pull + git lfs pull + + assert_pointer "master" "a.dat" "$oid" 1 + assert_same_inode "$TRASHDIR/$repo" "$TRASHDIR/$ref_repo" "$oid" +) +end_test diff --git a/test/test-worktree.sh b/test/test-worktree.sh index 3022ab7d..4fd8ac7d 100755 --- a/test/test-worktree.sh +++ b/test/test-worktree.sh @@ -24,6 +24,7 @@ LocalWorkingDir=$(native_path_escaped "$TRASHDIR/$reponame") LocalGitDir=$(native_path_escaped "$TRASHDIR/$reponame/.git") LocalGitStorageDir=$(native_path_escaped "$TRASHDIR/$reponame/.git") LocalMediaDir=$(native_path_escaped "$TRASHDIR/$reponame/.git/lfs/objects") +LocalReferenceDir= TempDir=$(native_path_escaped "$TRASHDIR/$reponame/.git/lfs/tmp") ConcurrentTransfers=3 BatchTransfer=true @@ -45,6 +46,7 @@ LocalWorkingDir=$(native_path_escaped "$TRASHDIR/$worktreename") LocalGitDir=$(native_path_escaped "$TRASHDIR/$reponame/.git/worktrees/$worktreename") LocalGitStorageDir=$(native_path_escaped "$TRASHDIR/$reponame/.git") LocalMediaDir=$(native_path_escaped "$TRASHDIR/$reponame/.git/lfs/objects") +LocalReferenceDir= TempDir=$(native_path_escaped "$TRASHDIR/$reponame/.git/worktrees/$worktreename/lfs/tmp") ConcurrentTransfers=3 BatchTransfer=true