lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for reference clone

Browse Source
This commit is contained in:
Jukka Lehtniemi 2016-01-31 23:06:40 +02:00
parent 95272fef40
commit 5069ecdfce
7 changed files with 190 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
commands/command_fetch.go
View File

@ -282,6 +282,15 @@ func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []str
// This avoids previous case of over-reporting a requirement for files we already have
// which would only be skipped by PointerSmudgeObject later
passFilter := lfs.FilenamePassesIncludeExcludeFilter(p.Name, include, exclude)
if !lfs.ObjectExistsOfSize(p.Oid, p.Size) {
altMediafile := lfs.LocalReferencePath(p.Oid)
mediafile, err := lfs.LocalMediaPath(p.Oid)
if err == nil && altMediafile != "" && lfs.FileExistsOfSize(altMediafile, p.Size) {
lfs.LinkOrCopy(altMediafile, mediafile)
}
}
if !lfs.ObjectExistsOfSize(p.Oid, p.Size) && passFilter {
tracerx.Printf("fetch %v [%v]", p.Name, p.Oid)
q.Add(lfs.NewDownloadable(p))

25
lfs/lfs.go
View File

@ -31,6 +31,7 @@ var (
LocalGitStorageDir string // parent of objects/lfs (may be same as LocalGitDir but may not)
LocalMediaDir string // root of lfs objects
LocalObjectTempDir string // where temporarily downloading objects are stored
LocalReferenceDir string // alternative local media dir (relative to clone reference repo)
objects *localstorage.LocalStorage
LocalLogDir string
checkedTempDir string
@ -56,6 +57,13 @@ func LocalMediaPath(oid string) (string, error) {
return objects.BuildObjectPath(oid)
}
func LocalReferencePath(sha string) string {
if LocalReferenceDir == "" {
return ""
}
return filepath.Join(LocalReferenceDir, sha[0:2], sha[2:4], sha)
}
func ObjectExistsOfSize(oid string, size int64) bool {
path := objects.ObjectPath(oid)
return FileExistsOfSize(path, size)
@ -69,6 +77,7 @@ func Environ() []string {
fmt.Sprintf("LocalGitDir=%s", LocalGitDir),
fmt.Sprintf("LocalGitStorageDir=%s", LocalGitStorageDir),
fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir),
fmt.Sprintf("LocalReferenceDir=%s", LocalReferenceDir),
fmt.Sprintf("TempDir=%s", TempDir),
fmt.Sprintf("ConcurrentTransfers=%d", Config.ConcurrentTransfers()),
fmt.Sprintf("BatchTransfer=%v", Config.BatchTransfer()),
@ -98,6 +107,7 @@ func ResolveDirs() {
LocalWorkingDir = ResolveSymlinks(LocalWorkingDir)
LocalGitStorageDir = resolveGitStorageDir(LocalGitDir)
LocalReferenceDir = resolveReferenceDir(LocalGitStorageDir)
TempDir = filepath.Join(LocalGitDir, "lfs", "tmp") // temp files per worktree
objs, err := localstorage.New(
@ -182,6 +192,21 @@ func processGitRedirectFile(file, prefix string) (string, error) {
return dir, nil
}
func resolveReferenceDir(gitStorageDir string) string {
cloneReferencePath := filepath.Join(gitStorageDir, "objects", "info", "alternates")
if FileExists(cloneReferencePath) {
buffer, err := ioutil.ReadFile(cloneReferencePath)
if err == nil {
path := strings.TrimSpace(string(buffer[:]))
referenceLfsStoragePath := filepath.Join(filepath.Dir(path), "lfs", "objects")
if DirExists(referenceLfsStoragePath) {
return referenceLfsStoragePath
}
}
}
return ""
}
// From a git dir, get the location that objects are to be stored (we will store lfs alongside)
// Sometimes there is an additional level of redirect on the .git folder by way of a commondir file
// before you find object storage, e.g. 'git worktree' uses this. It redirects to gitdir either by GIT_DIR

11
lfs/pointer_smudge.go
View File

@ -41,6 +41,17 @@ func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, download
}
stat, statErr := os.Stat(mediafile)
if statErr != nil || stat == nil {
altMediafile := LocalReferencePath(ptr.Oid)
if altMediafile != "" && FileExistsOfSize(altMediafile, ptr.Size) {
copyErr := LinkOrCopy(altMediafile, mediafile)
if copyErr == nil {
stat, statErr = os.Stat(mediafile)
}
}
}
if statErr == nil && stat != nil {
fileSize := stat.Size()
if fileSize == 0 || fileSize != ptr.Size {

32
lfs/util.go
View File

@ -326,3 +326,35 @@ func FileExistsOfSize(path string, sz int64) bool {
return !fi.IsDir() && fi.Size() == sz
}
func CopyFileContents(src string, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return err
}
defer func() {
cerr := out.Close()
if err == nil {
err = cerr
}
}()
_, err = io.Copy(out, in)
if err != nil {
return err
}
err = out.Sync()
return err
}
func LinkOrCopy(src string, dst string) error {
err := os.Link(src, dst)
if err == nil {
return err
}
return CopyFileContents(src, dst)
}

14
test/test-env.sh
View File

@ -27,6 +27,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -63,6 +64,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -106,6 +108,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -147,6 +150,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -190,6 +194,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -234,6 +239,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -280,6 +286,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=5
BatchTransfer=false
@ -332,6 +339,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -378,6 +386,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -417,6 +426,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -447,6 +457,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -466,6 +477,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -484,6 +496,7 @@ LocalWorkingDir=%s
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true
@ -513,6 +526,7 @@ LocalWorkingDir=
LocalGitDir=%s
LocalGitStorageDir=%s
LocalMediaDir=%s
LocalReferenceDir=
TempDir=%s
ConcurrentTransfers=3
BatchTransfer=true

97
test/test-reference-clone.sh Executable file
View File

@ -0,0 +1,97 @@
#!/usr/bin/env bash
. "test/testlib.sh"
assert_same_inode() {
local repo1=$1
local repo2=$2
local oid=$3
if ! uname -s | grep -qE 'CYGWIN|MSYS|MINGW'; then
cfg1=$(cd "$repo1"; git lfs env | grep LocalMediaDir)
f1="${cfg1:14}/${oid:0:2}/${oid:2:2}/$oid"
inode1=$(ls -i $f1 | cut -f1 -d\ )
cfg2=$(cd "$repo2"; git lfs env | grep LocalMediaDir)
f2="${cfg2:14}/${oid:0:2}/${oid:2:2}/$oid"
inode2=$(ls -i $f2 | cut -f1 -d\ )
[ "$inode1" == "$inode2" ]
fi
}
begin_test "clone with reference"
(
set -e
reponame="$(basename "$0" ".sh")"
setup_remote_repo "$reponame"
ref_repo=clone_reference_repo
ref_repo_dir=$TRASHDIR/$ref_repo
clone_repo "$reponame" "$ref_repo"
git lfs track "*.dat"
contents="a"
oid=$(calc_oid "$contents")
printf "$contents" > a.dat
git add a.dat
git add .gitattributes
git commit -m "add a.dat" 2>&1
git push origin master
delete_server_object "$reponame" "$oid"
repo=test_repo
repo_dir=$TRASHDIR/$repo
git clone --reference "$ref_repo_dir/.git" \
"$GITSERVER/$reponame" "$repo_dir" 2> clone.log
if grep -q "Downloading a.dat" clone.log; then
exit 1
fi
cd "$TRASHDIR/$repo"
assert_pointer "master" "a.dat" "$oid" 1
assert_same_inode "$repo_dir" "$ref_repo_dir" "$oid"
)
end_test
begin_test "fetch from clone reference"
(
set -e
reponame="$(basename "$0" ".sh")2"
setup_remote_repo "$reponame"
ref_repo=clone_reference_repo2
ref_repo_dir=$TRASHDIR/$ref_repo
clone_repo "$reponame" "$ref_repo"
repo=test_repo2
repo_dir=$TRASHDIR/$repo
git clone --reference "$ref_repo_dir/.git" \
"$GITSERVER/$reponame" "$repo_dir" 2> clone.log
cd "$ref_repo_dir"
git lfs track "*.dat"
contents="a"
oid=$(calc_oid "$contents")
printf "$contents" > a.dat
git add a.dat
git add .gitattributes
git commit -m "add a.dat" 2>&1
git push origin master
delete_server_object "$reponame" "$oid"
cd "$repo_dir"
GIT_LFS_SKIP_SMUDGE=1 git pull
git lfs pull
assert_pointer "master" "a.dat" "$oid" 1
assert_same_inode "$TRASHDIR/$repo" "$TRASHDIR/$ref_repo" "$oid"
)
end_test

2
test/test-worktree.sh
View File

@ -24,6 +24,7 @@ LocalWorkingDir=$(native_path_escaped "$TRASHDIR/$reponame")
LocalGitDir=$(native_path_escaped "$TRASHDIR/$reponame/.git")
LocalGitStorageDir=$(native_path_escaped "$TRASHDIR/$reponame/.git")
LocalMediaDir=$(native_path_escaped "$TRASHDIR/$reponame/.git/lfs/objects")
LocalReferenceDir=
TempDir=$(native_path_escaped "$TRASHDIR/$reponame/.git/lfs/tmp")
ConcurrentTransfers=3
BatchTransfer=true
@ -45,6 +46,7 @@ LocalWorkingDir=$(native_path_escaped "$TRASHDIR/$worktreename")
LocalGitDir=$(native_path_escaped "$TRASHDIR/$reponame/.git/worktrees/$worktreename")
LocalGitStorageDir=$(native_path_escaped "$TRASHDIR/$reponame/.git")
LocalMediaDir=$(native_path_escaped "$TRASHDIR/$reponame/.git/lfs/objects")
LocalReferenceDir=
TempDir=$(native_path_escaped "$TRASHDIR/$reponame/.git/worktrees/$worktreename/lfs/tmp")
ConcurrentTransfers=3
BatchTransfer=true