lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #527 from sinbad/fetch-pull-checkout

Browse Source 
Add checkout & pull commands, changes and fixes to fetch
This commit is contained in:
risk danger olson 2015-07-29 19:19:49 -06:00
commit fa57c152f5
17 changed files with 942 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
commands/command_checkout.go Normal file
View File

@ -0,0 +1,182 @@
package commands
import (
"os"
"os/exec"
"sync"
"github.com/github/git-lfs/git"
"github.com/github/git-lfs/lfs"
"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)
var (
checkoutCmd = &cobra.Command{
Use: "checkout",
Short: "Checks out LFS files into the working copy",
Run: checkoutCommand,
}
)
func checkoutCommand(cmd *cobra.Command, args []string) {
// Parameters are filters
// firstly convert any pathspecs to the root of the repo, in case this is being executed in a sub-folder
var rootedpaths = make([]string, len(args))
inchan := make(chan string, 1)
outchan, err := lfs.ConvertCwdFilesRelativeToRepo(inchan)
if err != nil {
Panic(err, "Could not checkout")
}
for _, arg := range args {
inchan <- arg
rootedpaths = append(rootedpaths, <-outchan)
}
close(inchan)
checkoutWithIncludeExclude(rootedpaths, nil)
}
func init() {
RootCmd.AddCommand(checkoutCmd)
}
// Checkout from items reported from the fetch process (in parallel)
func checkoutAllFromFetchChan(c chan *lfs.WrappedPointer) {
tracerx.Printf("starting fetch/parallel checkout")
checkoutFromFetchChan(nil, nil, c)
}
func checkoutFromFetchChan(include []string, exclude []string, in chan *lfs.WrappedPointer) {
ref, err := git.CurrentRef()
if err != nil {
Panic(err, "Could not checkout")
}
// Need to ScanTree to identify multiple files with the same content (fetch will only report oids once)
pointers, err := lfs.ScanTree(ref)
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
// Map oid to multiple pointers
mapping := make(map[string][]*lfs.WrappedPointer)
for _, pointer := range pointers {
if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
mapping[pointer.Oid] = append(mapping[pointer.Oid], pointer)
}
}
// Launch git update-index
c := make(chan *lfs.WrappedPointer)
var wait sync.WaitGroup
wait.Add(1)
go func() {
checkoutWithChan(c)
wait.Done()
}()
// Feed it from in, which comes from fetch
for p := range in {
// Add all of the files for this oid
for _, fp := range mapping[p.Oid] {
c <- fp
}
}
close(c)
wait.Wait()
}
func checkoutWithIncludeExclude(include []string, exclude []string) {
ref, err := git.CurrentRef()
if err != nil {
Panic(err, "Could not checkout")
}
pointers, err := lfs.ScanTree(ref)
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
var wait sync.WaitGroup
wait.Add(1)
c := make(chan *lfs.WrappedPointer)
go func() {
checkoutWithChan(c)
wait.Done()
}()
for _, pointer := range pointers {
if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
c <- pointer
}
}
close(c)
wait.Wait()
}
func checkoutAll() {
checkoutWithIncludeExclude(nil, nil)
}
// Populate the working copy with the real content of objects where the file is
// either missing, or contains a matching pointer placeholder, from a list of pointers.
// If the file exists but has other content it is left alone
func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
// Fire up the update-index command
cmd := exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
updateIdxStdin, err := cmd.StdinPipe()
if err != nil {
Panic(err, "Could not update the index")
}
if err := cmd.Start(); err != nil {
Panic(err, "Could not update the index")
}
// Get a converter from repo-relative to cwd-relative
// Since writing data & calling git update-index must be relative to cwd
repopathchan := make(chan string, 1)
cwdpathchan, err := lfs.ConvertRepoFilesRelativeToCwd(repopathchan)
if err != nil {
Panic(err, "Could not convert file paths")
}
// As files come in, write them to the wd and update the index
for pointer := range in {
// Check the content - either missing or still this pointer (not exist is ok)
filepointer, err := lfs.DecodePointerFromFile(pointer.Name)
if err != nil && !os.IsNotExist(err) {
if err == lfs.NotAPointerError {
// File has non-pointer content, leave it alone
continue
}
Panic(err, "Problem accessing %v", pointer.Name)
}
if filepointer != nil && filepointer.Oid != pointer.Oid {
// User has probably manually reset a file to another commit
// while leaving it a pointer; don't mess with this
continue
}
// OK now we can (over)write the file content
repopathchan <- pointer.Name
cwdfilepath := <-cwdpathchan
err = lfs.PointerSmudgeToFile(cwdfilepath, pointer.Pointer, nil)
if err != nil {
Panic(err, "Could not checkout file")
}
updateIdxStdin.Write([]byte(cwdfilepath + "\n"))
}
close(repopathchan)
updateIdxStdin.Close()
if err := cmd.Wait(); err != nil {
Panic(err, "Error updating the git index")
}
}

171
commands/command_fetch.go
View File

@ -1,9 +1,6 @@
package commands
import (
"os"
"os/exec"
"sync"
"time"
"github.com/github/git-lfs/git"
@ -15,7 +12,7 @@ import (
var (
fetchCmd = &cobra.Command{
Use: "fetch",
Short: "fetch",
Short: "Downloads LFS files",
Run: fetchCommand,
}
)
@ -33,94 +30,88 @@ func fetchCommand(cmd *cobra.Command, args []string) {
}
}
pointers, err := lfs.ScanRefs(ref, "", nil)
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
q := lfs.NewDownloadQueue(lfs.Config.ConcurrentTransfers(), len(pointers), totalSize)
for _, p := range pointers {
q.Add(lfs.NewDownloadable(p))
}
target, err := git.ResolveRef(ref)
if err != nil {
Panic(err, "Could not resolve git ref")
}
current, err := git.CurrentRef()
if err != nil {
Panic(err, "Could not fetch the current git ref")
}
var wait sync.WaitGroup
wait.Add(1)
if target == current {
// We just downloaded the files for the current ref, we can copy them into
// the working directory and update the git index. We're doing this in a
// goroutine so they can be copied as they come in, for efficiency.
watch := q.Watch()
go func() {
files := make(map[string]*lfs.WrappedPointer, len(pointers))
for _, pointer := range pointers {
files[pointer.Oid] = pointer
}
// Fire up the update-index command
cmd := exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
stdin, err := cmd.StdinPipe()
if err != nil {
Panic(err, "Could not update the index")
}
if err := cmd.Start(); err != nil {
Panic(err, "Could not update the index")
}
// As files come in, write them to the wd and update the index
for oid := range watch {
pointer, ok := files[oid]
if !ok {
continue
}
file, err := os.Create(pointer.Name)
if err != nil {
Panic(err, "Could not create working directory file")
}
if err := lfs.PointerSmudge(file, pointer.Pointer, pointer.Name, nil); err != nil {
Panic(err, "Could not write working directory file")
}
file.Close()
stdin.Write([]byte(pointer.Name + "\n"))
}
stdin.Close()
if err := cmd.Wait(); err != nil {
Panic(err, "Error updating the git index")
}
wait.Done()
}()
} else {
wait.Done()
}
processQueue := time.Now()
q.Wait()
tracerx.PerformanceSince("process queue", processQueue)
wait.Wait()
fetchRef(ref)
}
func init() {
RootCmd.AddCommand(fetchCmd)
}
func fetchRefToChan(ref string) chan *lfs.WrappedPointer {
c := make(chan *lfs.WrappedPointer)
pointers, err := lfs.ScanRefs(ref, "", nil)
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
go fetchAndReportToChan(pointers, c)
return c
}
// Fetch all binaries for a given ref (that we don't have already)
func fetchRef(ref string) {
pointers, err := lfs.ScanRefs(ref, "", nil)
if err != nil {
Panic(err, "Could not scan for Git LFS files")
}
fetchPointers(pointers)
}
func fetchPointers(pointers []*lfs.WrappedPointer) {
fetchAndReportToChan(pointers, nil)
}
// Fetch and report completion of each OID to a channel (optional, pass nil to skip)
func fetchAndReportToChan(pointers []*lfs.WrappedPointer, out chan<- *lfs.WrappedPointer) {
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}
q := lfs.NewDownloadQueue(lfs.Config.ConcurrentTransfers(), len(pointers), totalSize)
for _, p := range pointers {
// Only add to download queue if local file is not the right size already
// This avoids previous case of over-reporting a requirement for files we already have
// which would only be skipped by PointerSmudgeObject later
if !lfs.ObjectExistsOfSize(p.Oid, p.Size) {
q.Add(lfs.NewDownloadable(p))
} else {
// If we already have it, report it to chan immediately to support pull/checkout
if out != nil {
out <- p
}
}
}
if out != nil {
dlwatch := q.Watch()
go func() {
// fetch only reports single OID, but OID *might* be referenced by multiple
// WrappedPointers if same content is at multiple paths, so map oid->slice
oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers))
for _, pointer := range pointers {
plist := oidToPointers[pointer.Oid]
oidToPointers[pointer.Oid] = append(plist, pointer)
}
for oid := range dlwatch {
plist, ok := oidToPointers[oid]
if !ok {
continue
}
for _, p := range plist {
out <- p
}
}
close(out)
}()
}
processQueue := time.Now()
q.Wait()
tracerx.PerformanceSince("process queue", processQueue)
}

29
commands/command_pull.go Normal file
View File

@ -0,0 +1,29 @@
package commands
import (
"github.com/github/git-lfs/git"
"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)
var (
pullCmd = &cobra.Command{
Use: "pull",
Short: "Downloads LFS files for the current ref, and checks out",
Run: pullCommand,
}
)
func pullCommand(cmd *cobra.Command, args []string) {
ref, err := git.CurrentRef()
if err != nil {
Panic(err, "Could not pull")
}
c := fetchRefToChan(ref)
checkoutAllFromFetchChan(c)
}
func init() {
RootCmd.AddCommand(pullCmd)
}

36
docs/man/git-lfs-checkout.1.ronn Normal file
View File

@ -0,0 +1,36 @@
git-lfs-checkout(1) -- Update working copy with file content if available
=========================================================================
## SYNOPSIS
`git lfs checkout` <filespec>...
## DESCRIPTION
Try to ensure that the working copy contains file content for Git LFS objects
for the current ref, if the object data is available. Does not download any
content, see git-lfs-fetch(1) for that.
Checkout scans the current ref for all LFS objects that would be required, then
where a file is either missing in the working copy, or contains placeholder
pointer content with the same SHA, the real file content is written, provided
we have it in the local store. Modified files are never overwritten.
Filespecs can be provided as arguments to restrict the files which are updated.
## EXAMPLES
* Checkout all files that are missing or placeholders
`git lfs checkout`
* Checkout a specific couple of files
`git lfs checkout path/to/file1.png path/to.file2.png`
## SEE ALSO
git-lfs-fetch(1), git-lfs-pull(1).
Part of the git-lfs(1) suite.

7
docs/man/git-lfs-fetch.1.ronn
View File

@ -10,8 +10,7 @@ git-lfs-fetch(1) -- Download all Git LFS files for a given ref
Download any Git LFS objects for a given ref. If no ref is given,
the currently checked out ref will be used.
If the given ref is the same as the currently checked out ref, the
files will be written to the working directory.
This does not update the working copy.
## EXAMPLES
@ -27,5 +26,9 @@ files will be written to the working directory.
`git lfs fetch e445b45c1c9c6282614f201b62778e4c0688b5c8`
## SEE ALSO
git-lfs-checkout(1), git-lfs-pull(1).
Part of the git-lfs(1) suite.

25
docs/man/git-lfs-pull.1.ronn Normal file
View File

@ -0,0 +1,25 @@
git-lfs-pull(1) -- Download all Git LFS files for current ref & checkout
========================================================================
## SYNOPSIS
`git lfs pull`
## DESCRIPTION
Download Git LFS objects for the currently checked out ref, and update
the working copy with the downloaded content if required.
This is equivalent to running the following 2 commands:
git lfs fetch
git lfs checkout
## EXAMPLES
## SEE ALSO
git-lfs-fetch(1), git-lfs-checkout(1).
Part of the git-lfs(1) suite.

18
lfs/lfs.go
View File

@ -46,8 +46,15 @@ func ResetTempDir() error {
return os.RemoveAll(TempDir)
}
func localMediaDirNoCreate(sha string) string {
return filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
}
func localMediaPathNoCreate(sha string) string {
return filepath.Join(localMediaDirNoCreate(sha), sha)
}
func LocalMediaPath(sha string) (string, error) {
path := filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
path := localMediaDirNoCreate(sha)
if err := os.MkdirAll(path, localMediaDirPerms); err != nil {
return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
}
@ -55,6 +62,15 @@ func LocalMediaPath(sha string) (string, error) {
return filepath.Join(path, sha), nil
}
func ObjectExistsOfSize(sha string, size int64) bool {
path := localMediaPathNoCreate(sha)
stat, err := os.Stat(path)
if err == nil && size == stat.Size() {
return true
}
return false
}
func Environ() []string {
osEnviron := os.Environ()
env := make([]string, 6, len(osEnviron)+6)

21
lfs/pointer.go
View File

@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
"os"
"regexp"
"strconv"
"strings"
@ -27,6 +28,8 @@ size %d
`
matcherRE = regexp.MustCompile("git-media|hawser|git-lfs")
pointerKeys = []string{"version", "oid", "size"}
NotAPointerError = errors.New("Not a valid Git LFS pointer file.")
)
type Pointer struct {
@ -56,6 +59,22 @@ func EncodePointer(writer io.Writer, pointer *Pointer) (int, error) {
return writer.Write([]byte(pointer.Encoded()))
}
func DecodePointerFromFile(file string) (*Pointer, error) {
// Check size before reading
stat, err := os.Stat(file)
if err != nil {
return nil, err
}
if stat.Size() > blobSizeCutoff {
return nil, NotAPointerError
}
f, err := os.OpenFile(file, os.O_RDONLY, 0644)
if err != nil {
return nil, err
}
defer f.Close()
return DecodePointer(f)
}
func DecodePointer(reader io.Reader) (*Pointer, error) {
_, p, err := DecodeFrom(reader)
return p, err
@ -130,7 +149,7 @@ func decodeKVData(data []byte) (map[string]string, error) {
m := make(map[string]string)
if !matcherRE.Match(data) {
return m, fmt.Errorf("Not a valid Git LFS pointer file.")
return m, NotAPointerError
}
scanner := bufio.NewScanner(bytes.NewBuffer(data))

12
lfs/pointer_smudge.go
View File

@ -11,6 +11,18 @@ import (
contentaddressable "github.com/github/git-lfs/vendor/_nuts/github.com/technoweenie/go-contentaddressable"
)
func PointerSmudgeToFile(filename string, ptr *Pointer, cb CopyCallback) error {
os.MkdirAll(filepath.Dir(filename), 0755)
file, err := os.Create(filename)
if err != nil {
return fmt.Errorf("Could not create working directory file: %v", err)
}
defer file.Close()
if err := PointerSmudge(file, ptr, filename, cb); err != nil {
return fmt.Errorf("Could not write working directory file: %v", err)
}
return nil
}
func PointerSmudge(writer io.Writer, ptr *Pointer, workingfile string, cb CopyCallback) error {
mediafile, err := LocalMediaPath(ptr.Oid)
if err != nil {

4
lfs/progress_meter.go
View File

@ -84,7 +84,7 @@ func (p *ProgressMeter) Finish() {
close(p.finished)
p.update()
p.logger.Close()
if p.show {
if p.show && p.estimatedBytes > 0 {
fmt.Fprintf(os.Stdout, "\n")
}
}
@ -116,7 +116,7 @@ func (p *ProgressMeter) writer() {
}
func (p *ProgressMeter) update() {
if !p.show {
if !p.show || p.estimatedFiles == 0 {
return
}

133
lfs/scanner.go
View File

@ -57,6 +57,7 @@ type ScanRefsOptions struct {
// ScanRefs takes a ref and returns a slice of WrappedPointer objects
// for all Git LFS pointers it finds for that ref.
// Reports unique oids once only, not multiple times if >1 file uses the same content
func ScanRefs(refLeft, refRight string, opt *ScanRefsOptions) ([]*WrappedPointer, error) {
if opt == nil {
opt = &ScanRefsOptions{}
@ -97,6 +98,7 @@ func ScanRefs(refLeft, refRight string, opt *ScanRefsOptions) ([]*WrappedPointer
// ScanIndex returns a slice of WrappedPointer objects for all
// Git LFS pointers it finds in the index.
// Reports unique oids once only, not multiple times if >1 file uses the same content
func ScanIndex() ([]*WrappedPointer, error) {
nameMap := make(map[string]*indexFile, 0)
@ -384,3 +386,134 @@ func startCommand(command string, args ...string) (*wrappedCmd, error) {
return &wrappedCmd{stdin, bufio.NewReaderSize(stdout, stdoutBufSize), cmd}, nil
}
// An entry from ls-tree or rev-list including a blob sha and tree path
type TreeBlob struct {
Sha1 string
Filename string
}
// ScanTree takes a ref and returns a slice of WrappedPointer objects in the tree at that ref
// Differs from ScanRefs in that multiple files in the tree with the same content are all reported
func ScanTree(ref string) ([]*WrappedPointer, error) {
start := time.Now()
defer func() {
tracerx.PerformanceSince("scan", start)
}()
// We don't use the nameMap approach here since that's imprecise when >1 file
// can be using the same content
treeShas, err := lsTreeBlobs(ref)
if err != nil {
return nil, err
}
pointerc, err := catFileBatchTree(treeShas)
if err != nil {
return nil, err
}
pointers := make([]*WrappedPointer, 0)
for p := range pointerc {
pointers = append(pointers, p)
}
return pointers, nil
}
// catFileBatchTree uses git cat-file --batch to get the object contents
// of a git object, given its sha1. The contents will be decoded into
// a Git LFS pointer. treeblobs is a channel over which blob entries
// will be sent. It returns a channel from which point.Pointers can be read.
func catFileBatchTree(treeblobs chan TreeBlob) (chan *WrappedPointer, error) {
cmd, err := startCommand("git", "cat-file", "--batch")
if err != nil {
return nil, err
}
pointers := make(chan *WrappedPointer, chanBufSize)
go func() {
for t := range treeblobs {
cmd.Stdin.Write([]byte(t.Sha1 + "\n"))
l, err := cmd.Stdout.ReadBytes('\n')
if err != nil {
break
}
// Line is formatted:
// <sha1> <type> <size>
fields := bytes.Fields(l)
s, _ := strconv.Atoi(string(fields[2]))
nbuf := make([]byte, s)
_, err = io.ReadFull(cmd.Stdout, nbuf)
if err != nil {
break // Legit errors
}
p, err := DecodePointer(bytes.NewBuffer(nbuf))
if err == nil {
pointers <- &WrappedPointer{
Sha1: string(fields[0]),
Size: p.Size,
Pointer: p,
Name: t.Filename,
}
}
_, err = cmd.Stdout.ReadBytes('\n') // Extra \n inserted by cat-file
if err != nil {
break
}
}
close(pointers)
cmd.Stdin.Close()
}()
return pointers, nil
}
// Use ls-tree at ref to find a list of candidate tree blobs which might be lfs files
// The returned channel will be sent these blobs which should be sent to catFileBatchTree
// for final check & conversion to Pointer
func lsTreeBlobs(ref string) (chan TreeBlob, error) {
// Snapshot using ls-tree
lsArgs := []string{"ls-tree",
"-r", // recurse
"-l", // report object size (we'll need this)
"--full-tree", // start at the root regardless of where we are in it
ref}
cmd, err := startCommand("git", lsArgs...)
if err != nil {
return nil, err
}
cmd.Stdin.Close()
blobs := make(chan TreeBlob, chanBufSize)
go func() {
scanner := bufio.NewScanner(cmd.Stdout)
regex := regexp.MustCompile(`^\d+\s+blob\s+([0-9a-zA-Z]{40})\s+(\d+)\s+(.*)$`)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if match := regex.FindStringSubmatch(line); match != nil {
sz, err := strconv.ParseInt(match[2], 10, 64)
if err != nil {
continue
}
sha1 := match[1]
filename := match[3]
if sz < blobSizeCutoff {
blobs <- TreeBlob{sha1, filename}
}
}
}
close(blobs)
}()
return blobs, nil
}

174
lfs/util.go
View File

@ -5,6 +5,8 @@ import (
"io"
"os"
"path/filepath"
"runtime"
"strings"
)
type CallbackReader struct {
@ -14,6 +16,18 @@ type CallbackReader struct {
io.Reader
}
type Platform int
const (
PlatformWindows = Platform(iota)
PlatformLinux = Platform(iota)
PlatformOSX = Platform(iota)
PlatformOther = Platform(iota) // most likely a *nix variant e.g. freebsd
PlatformUndetermined = Platform(iota)
)
var currentPlatform = PlatformUndetermined
type CopyCallback func(totalSize int64, readSoFar int64, readSinceLast int) error
func (w *CallbackReader) Read(p []byte) (int, error) {
@ -86,3 +100,163 @@ func wrapProgressError(err error, event, filename string) error {
return nil
}
// Return whether a given filename passes the include / exclude path filters
// Only paths that are in includePaths and outside excludePaths are passed
// If includePaths is empty that filter always passes and the same with excludePaths
// Both path lists support wildcard matches
func FilenamePassesIncludeExcludeFilter(filename string, includePaths, excludePaths []string) bool {
if len(includePaths) == 0 && len(excludePaths) == 0 {
return true
}
// For Win32, because git reports files with / separators
cleanfilename := filepath.Clean(filename)
if len(includePaths) > 0 {
matched := false
for _, inc := range includePaths {
matched, _ = filepath.Match(inc, filename)
if !matched && IsWindows() {
// Also Win32 match
matched, _ = filepath.Match(inc, cleanfilename)
}
if !matched {
// Also support matching a parent directory without a wildcard
if strings.HasPrefix(cleanfilename, inc+string(filepath.Separator)) {
matched = true
}
}
if matched {
break
}
}
if !matched {
return false
}
}
if len(excludePaths) > 0 {
for _, ex := range excludePaths {
matched, _ := filepath.Match(ex, filename)
if !matched && IsWindows() {
// Also Win32 match
matched, _ = filepath.Match(ex, cleanfilename)
}
if matched {
return false
}
// Also support matching a parent directory without a wildcard
if strings.HasPrefix(cleanfilename, ex+string(filepath.Separator)) {
return false
}
}
}
return true
}
func GetPlatform() Platform {
if currentPlatform == PlatformUndetermined {
switch runtime.GOOS {
case "windows":
currentPlatform = PlatformWindows
case "linux":
currentPlatform = PlatformLinux
case "darwin":
currentPlatform = PlatformOSX
default:
currentPlatform = PlatformOther
}
}
return currentPlatform
}
// Convert filenames expressed relative to the root of the repo relative to the
// current working dir. Useful when needing to calling git with results from a rooted command,
// but the user is in a subdir of their repo
// Pass in a channel which you will fill with relative files & receive a channel which will get results
func ConvertRepoFilesRelativeToCwd(repochan <-chan string) (<-chan string, error) {
wd, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("Unable to get working dir: %v", err)
}
// Early-out if working dir is root dir, same result
passthrough := false
if LocalWorkingDir == wd {
passthrough = true
}
outchan := make(chan string, 1)
go func() {
for f := range repochan {
if passthrough {
outchan <- f
continue
}
abs := filepath.Join(LocalWorkingDir, f)
rel, err := filepath.Rel(wd, abs)
if err != nil {
// Use absolute file instead
outchan <- abs
} else {
outchan <- rel
}
}
close(outchan)
}()
return outchan, nil
}
// Convert filenames expressed relative to the current directory to be
// relative to the repo root. Useful when calling git with arguments that requires them
// to be rooted but the user is in a subdir of their repo & expects to use relative args
// Pass in a channel which you will fill with relative files & receive a channel which will get results
func ConvertCwdFilesRelativeToRepo(cwdchan <-chan string) (<-chan string, error) {
curdir, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("Could not retrieve current directory: %v", err)
}
// Early-out if working dir is root dir, same result
passthrough := false
if LocalWorkingDir == curdir {
passthrough = true
}
outchan := make(chan string, 1)
go func() {
for p := range cwdchan {
if passthrough {
outchan <- p
continue
}
var abs string
if filepath.IsAbs(p) {
abs = p
} else {
abs = filepath.Join(curdir, p)
}
reltoroot, err := filepath.Rel(LocalWorkingDir, abs)
if err != nil {
// Can't do this, use absolute as best fallback
outchan <- abs
} else {
outchan <- reltoroot
}
}
close(outchan)
}()
return outchan, nil
}
// Are we running on Windows? Need to handle some extra path shenanigans
func IsWindows() bool {
return GetPlatform() == PlatformWindows
}

77
lfs/util_test.go
View File

@ -57,3 +57,80 @@ func TestCopyWithCallback(t *testing.T) {
assert.Equal(t, 1, len(calledWritten))
assert.Equal(t, 5, int(calledWritten[0]))
}
func TestFilterIncludeExclude(t *testing.T) {
// Inclusion
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test/filename.dat"}, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"blank", "something", "test/filename.dat", "foo"}, nil))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"blank", "something", "foo"}, nil))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test/notfilename.dat"}, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test"}, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test/*"}, nil))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"nottest"}, nil))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"nottest/*"}, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test/fil*"}, nil))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test/g*"}, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"tes*/*"}, nil))
// Exclusion
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test/filename.dat"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"blank", "something", "test/filename.dat", "foo"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"blank", "something", "foo"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test/notfilename.dat"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test/*"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"nottest"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"nottest/*"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test/fil*"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test/g*"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"tes*/*"}))
// Both
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test/filename.dat"}, []string{"test/notfilename.dat"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test"}, []string{"test/filename.dat"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test/*"}, []string{"test/notfile*"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test/*"}, []string{"test/file*"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"another/*", "test/*"}, []string{"test/notfilename.dat", "test/filename.dat"}))
if IsWindows() {
// Extra tests because Windows git reports filenames with / separators
// but we need to allow \ separators in include/exclude too
// Can only test this ON Windows because of filepath behaviour
// Inclusion
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test\\filename.dat"}, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"blank", "something", "test\\filename.dat", "foo"}, nil))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"blank", "something", "foo"}, nil))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test\\notfilename.dat"}, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test"}, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test\\*"}, nil))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"nottest"}, nil))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"nottest\\*"}, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test\\fil*"}, nil))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test\\g*"}, nil))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"tes*\\*"}, nil))
// Exclusion
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test\\filename.dat"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"blank", "something", "test\\filename.dat", "foo"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"blank", "something", "foo"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test\\notfilename.dat"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test\\*"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"nottest"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"nottest\\*"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test\\fil*"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"test\\g*"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", nil, []string{"tes*\\*"}))
// Both
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test\\filename.dat"}, []string{"test\\notfilename.dat"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test"}, []string{"test\\filename.dat"}))
assert.Equal(t, true, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test\\*"}, []string{"test\\notfile*"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"test\\*"}, []string{"test\\file*"}))
assert.Equal(t, false, FilenamePassesIncludeExcludeFilter("test/filename.dat", []string{"another\\*", "test\\*"}, []string{"test\\notfilename.dat", "test\\filename.dat"}))
}
}

79
test/test-checkout.sh Executable file
View File

@ -0,0 +1,79 @@
#!/bin/bash
. "test/testlib.sh"
begin_test "checkout"
(
set -e
reponame="$(basename "$0" ".sh")"
setup_remote_repo "$reponame"
clone_repo "$reponame" repo
git lfs track "*.dat" 2>&1 | tee track.log
grep "Tracking \*.dat" track.log
contents="something something"
contents_oid=$(printf "$contents" | shasum -a 256 | cut -f 1 -d " ")
# Same content everywhere is ok, just one object in lfs db
printf "$contents" > file1.dat
printf "$contents" > file2.dat
printf "$contents" > file3.dat
mkdir folder1 folder2
printf "$contents" > folder1/nested.dat
printf "$contents" > folder2/nested.dat
git add file1.dat file2.dat file3.dat folder1/nested.dat folder2/nested.dat
git add .gitattributes
git commit -m "add files"
[ "$contents" = "$(cat file1.dat)" ]
[ "$contents" = "$(cat file2.dat)" ]
[ "$contents" = "$(cat file3.dat)" ]
[ "$contents" = "$(cat folder1/nested.dat)" ]
[ "$contents" = "$(cat folder2/nested.dat)" ]
assert_pointer "master" "file1.dat" "$contents_oid" 19
# Remove the working directory
rm -rf file1.dat file2.dat file3.dat folder1/nested.dat folder2/nested.dat
# checkout should replace all
git lfs checkout
[ "$contents" = "$(cat file1.dat)" ]
[ "$contents" = "$(cat file2.dat)" ]
[ "$contents" = "$(cat file3.dat)" ]
[ "$contents" = "$(cat folder1/nested.dat)" ]
[ "$contents" = "$(cat folder2/nested.dat)" ]
# Remove again
rm -rf file1.dat file2.dat file3.dat folder1/nested.dat folder2/nested.dat
# checkout with filters
git lfs checkout file2.dat
[ "$contents" = "$(cat file2.dat)" ]
[ ! -f file1.dat ]
[ ! -f file3.dat ]
[ ! -f folder1/nested.dat ]
[ ! -f folder2/nested.dat ]
# quotes to avoid shell globbing
git lfs checkout "file*.dat"
[ "$contents" = "$(cat file1.dat)" ]
[ "$contents" = "$(cat file3.dat)" ]
[ ! -f folder1/nested.dat ]
[ ! -f folder2/nested.dat ]
# test subdir context
pushd folder1
git lfs checkout nested.dat
[ "$contents" = "$(cat nested.dat)" ]
[ ! -f ../folder2/nested.dat ]
popd
# test folder param
git lfs checkout folder2
[ "$contents" = "$(cat folder2/nested.dat)" ]
)
end_test

3
test/test-fetch.sh
View File

@ -63,12 +63,11 @@ begin_test "fetch"
# Remove the working directory and lfs files
rm a.dat
rm -rf .git/lfs/objects
git lfs fetch 2>&1 | grep "(1 of 1 files)"
[ "a" = "$(cat a.dat)" ]
assert_pointer "master" "a.dat" "$contents_oid" 1
git checkout newbranch
git checkout master

5
test/test-pre-push.sh
View File

@ -1,4 +1,4 @@
#!/bin/sh
#!/bin/bash
. "test/testlib.sh"
@ -17,7 +17,8 @@ begin_test "pre-push"
echo "refs/heads/master master refs/heads/master 0000000000000000000000000000000000000000" |
git lfs pre-push origin "$GITSERVER/$reponame" 2>&1 |
tee push.log
grep "(0 of 0 files)" push.log
# no output if nothing to do
[ "$(du -k push.log | cut -f 1)" == "0" ]
git lfs track "*.dat"
echo "hi" > hi.dat

66
test/test-pull.sh Executable file
View File

@ -0,0 +1,66 @@
#!/bin/sh
. "test/testlib.sh"
begin_test "pull"
(
set -e
reponame="$(basename "$0" ".sh")"
setup_remote_repo "$reponame"
clone_repo "$reponame" clone
clone_repo "$reponame" repo
git lfs track "*.dat" 2>&1 | tee track.log
grep "Tracking \*.dat" track.log
contents="a"
contents_oid=$(printf "$contents" | shasum -a 256 | cut -f 1 -d " ")
printf "$contents" > a.dat
git add a.dat
git add .gitattributes
git commit -m "add a.dat" 2>&1 | tee commit.log
grep "master (root-commit)" commit.log
grep "2 files changed" commit.log
grep "create mode 100644 a.dat" commit.log
grep "create mode 100644 .gitattributes" commit.log
[ "a" = "$(cat a.dat)" ]
assert_pointer "master" "a.dat" "$contents_oid" 1
refute_server_object "$reponame" "$contents_oid"
git push origin master 2>&1 | tee push.log
grep "(1 of 1 files)" push.log
grep "master -> master" push.log
assert_server_object "$reponame" "$contents_oid"
# change to the clone's working directory
cd ../clone
git pull 2>&1 | grep "Downloading a.dat (1 B)"
[ "a" = "$(cat a.dat)" ]
assert_pointer "master" "a.dat" "$contents_oid" 1
# Remove the working directory and lfs files
rm a.dat
rm -rf .git/lfs/objects
git lfs pull 2>&1 | grep "(1 of 1 files)"
[ "a" = "$(cat a.dat)" ]
assert_pointer "master" "a.dat" "$contents_oid" 1
# Remove just the working directory
rm a.dat
git lfs pull
[ "a" = "$(cat a.dat)" ]
)
end_test