lunny/git-lfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into checkout-no-clean

Browse Source
This commit is contained in:
risk danger olson 2017-10-18 15:18:11 -07:00 committed by GitHub
commit cc3afc8225
31 changed files with 353 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
CHANGELOG.md
View File

@ -1,5 +1,42 @@
# Git LFS Changelog
## 2.3.4 (18 October, 2017)
### Features
* 'git lfs install' updates filters with 'skip-smudge' option #2673 (@technoweenie)
### Bugs
* FastWalkGitRepo: limit number of concurrent goroutines #2672 (@technoweenie)
* handle scenario where multiple configuration values exist in ~/.gitconfig #2659 (@shiftkey)
## 2.3.3 (9 October, 2017)
### Bugs
* invoke lfs for 'git update-index', fixing 'status' issues #2647 (@technoweenie)
* cache http credential helper output by default #2648 (@technoweenie)
## 2.3.2 (3 October, 2017)
### Features
* bump default activity timeout from 10s -> 30s #2632 (@technoweenie)
### Bugs
* ensure files are marked readonly after unlocking by ID #2642 (@technoweenie)
* add files to index with path relative to current dir #2641 (@technoweenie)
* better Netrc errors #2633 (@technoweenie)
* only use askpass if credential.helper is not configured #2637 (@technoweenie)
* convert backslash to slash when writing to .gitattributes #2625 (@technoweenie)
### Misc
* only copy req headers if there are git-configured extra headers #2622 (@technoweenie)
* update tracerx to add timestamps #2620 (@rubyist)
## 2.3.1 (27 September, 2017)
### Features

5
commands/command_install.go
View File

@ -25,8 +25,9 @@ func installCommand(cmd *cobra.Command, args []string) {
}
if err := lfs.InstallFilters(opt, skipSmudgeInstall); err != nil {
Error(err.Error())
Exit("Run `git lfs install --force` to reset git config.")
Print("WARNING: %s", err.Error())
Print("Run `git lfs install --force` to reset git config.")
return
}
if !skipRepoInstall && (localInstall || lfs.InRepo()) {

4
commands/command_track.go
View File

@ -97,7 +97,7 @@ ArgsLoop:
writeablePatterns = append(writeablePatterns, pattern)
}
Print("Tracking %q", pattern)
Print("Tracking %q", unescapeTrackPattern(encodedArg))
}
// Now read the whole local attributes file and iterate over the contents,
@ -261,7 +261,7 @@ var (
)
func escapeTrackPattern(unescaped string) string {
var escaped string = unescaped
var escaped string = strings.Replace(unescaped, `\`, "/", -1)
for from, to := range trackEscapePatterns {
escaped = strings.Replace(escaped, from, to, -1)

9
commands/commands.go
View File

@ -81,6 +81,15 @@ func getAPIClient() *lfsapi.Client {
return apiClient
}
func closeAPIClient() error {
global.Lock()
defer global.Unlock()
if apiClient == nil {
return nil
}
return apiClient.Close()
}
func newLockClient(remote string) *locking.Client {
storageConfig := config.Config.StorageConfig()
lockClient, err := locking.NewClient(remote, getAPIClient())

6
commands/pull.go
View File

@ -61,8 +61,10 @@ func (c *singleCheckout) Skip() bool {
}
func (c *singleCheckout) Run(p *lfs.WrappedPointer) {
cwdfilepath := c.pathConverter.Convert(p.Name)
// Check the content - either missing or still this pointer (not exist is ok)
filepointer, err := lfs.DecodePointerFromFile(p.Name)
filepointer, err := lfs.DecodePointerFromFile(cwdfilepath)
if err != nil && !os.IsNotExist(err) {
if errors.IsNotAPointerError(err) {
// File has non-pointer content, leave it alone
@ -79,8 +81,6 @@ func (c *singleCheckout) Run(p *lfs.WrappedPointer) {
return
}
cwdfilepath := c.pathConverter.Convert(p.Name)
err = lfs.PointerSmudgeToFile(cwdfilepath, p.Pointer, false, c.manifest, nil)
if err != nil {
if errors.IsDownloadDeclinedError(err) {

2
commands/run.go
View File

@ -66,7 +66,7 @@ func Run() {
}
root.Execute()
getAPIClient().Close()
closeAPIClient()
}
func gitlfsCommand(cmd *cobra.Command, args []string) {

2
config/version.go
View File

@ -12,7 +12,7 @@ var (
)
const (
Version = "2.3.1"
Version = "2.3.4"
)
func init() {

18
debian/changelog vendored
View File

@ -1,3 +1,21 @@
git-lfs (2.3.4) stable; urgency=low
* New upstream version
-- Rick Olson <technoweenie@gmail.com> Wed, 18 Oct 2017 14:29:00 +0000
git-lfs (2.3.3) stable; urgency=low
* New upstream version
-- Rick Olson <technoweenie@gmail.com> Mon, 9 Oct 2017 14:29:00 +0000
git-lfs (2.3.2) stable; urgency=low
* New upstream version
-- Rick Olson <technoweenie@gmail.com> Tue, 3 Oct 2017 14:29:00 +0000
git-lfs (2.3.1) stable; urgency=low
* New upstream version

2
docs/man/git-lfs-config.5.ronn
View File

@ -44,7 +44,7 @@ be scoped inside the configuration for a remote.
Sets the maximum time, in seconds, that the HTTP client will wait for the
next tcp read or write. If < 1, no activity timeout is used at all.
Default: 10 seconds
Default: 30 seconds
* `lfs.keepalive`

10
git/git.go
View File

@ -459,7 +459,7 @@ func DefaultRemote() (string, error) {
}
func UpdateIndexFromStdin() *subprocess.Cmd {
return gitNoLFS("update-index", "-q", "--refresh", "--stdin")
return git("update-index", "-q", "--refresh", "--stdin")
}
type gitConfig struct {
@ -495,12 +495,12 @@ func (c *gitConfig) FindLocal(val string) string {
// SetGlobal sets the git config value for the key in the global config
func (c *gitConfig) SetGlobal(key, val string) (string, error) {
return gitSimple("config", "--global", key, val)
return gitSimple("config", "--global", "--replace-all", key, val)
}
// SetSystem sets the git config value for the key in the system config
func (c *gitConfig) SetSystem(key, val string) (string, error) {
return gitSimple("config", "--system", key, val)
return gitSimple("config", "--system", "--replace-all", key, val)
}
// UnsetGlobal removes the git config value for the key from the global config
@ -530,12 +530,12 @@ func (c *gitConfig) UnsetLocalSection(key string) (string, error) {
// SetLocal sets the git config value for the key in the specified config file
func (c *gitConfig) SetLocal(file, key, val string) (string, error) {
args := make([]string, 1, 5)
args := make([]string, 1, 6)
args[0] = "config"
if len(file) > 0 {
args = append(args, "--file", file)
}
args = append(args, key, val)
args = append(args, "--replace-all", key, val)
return gitSimple(args...)
}

8
lfs/attribute.go
View File

@ -78,21 +78,15 @@ func (a *Attribute) set(key, value string, upgradeables []string, opt InstallOpt
if opt.Force || shouldReset(currentValue, upgradeables) {
var err error
if opt.Local {
// ignore error for unset, git returns non-zero if missing
git.Config.UnsetLocalKey("", key)
_, err = git.Config.SetLocal("", key, value)
} else if opt.System {
// ignore error for unset, git returns non-zero if missing
git.Config.UnsetSystem(key)
_, err = git.Config.SetSystem(key, value)
} else {
// ignore error for unset, git returns non-zero if missing
git.Config.UnsetGlobal(key)
_, err = git.Config.SetGlobal(key, value)
}
return err
} else if currentValue != value {
return fmt.Errorf("The %s attribute should be %q but is %q",
return fmt.Errorf("The %q attribute should be %q but is %q",
key, value, currentValue)
}

28
lfs/setup.go
View File

@ -28,6 +28,22 @@ var (
postMergeHook,
}
upgradeables = map[string][]string{
"clean": []string{"git-lfs clean %f"},
"smudge": []string{
"git-lfs smudge %f",
"git-lfs smudge --skip %f",
"git-lfs smudge -- %f",
"git-lfs smudge --skip -- %f",
},
"process": []string{
"git-lfs filter",
"git-lfs filter --skip",
"git-lfs filter-process",
"git-lfs filter-process --skip",
},
}
filters = &Attribute{
Section: "filter.lfs",
Properties: map[string]string{
@ -36,11 +52,7 @@ var (
"process": "git-lfs filter-process",
"required": "true",
},
Upgradeables: map[string][]string{
"clean": []string{"git-lfs clean %f"},
"smudge": []string{"git-lfs smudge %f"},
"process": []string{"git-lfs filter"},
},
Upgradeables: upgradeables,
}
passFilters = &Attribute{
@ -51,11 +63,7 @@ var (
"process": "git-lfs filter-process --skip",
"required": "true",
},
Upgradeables: map[string][]string{
"clean": []string{"git-lfs clean %f"},
"smudge": []string{"git-lfs smudge --skip %f"},
"process": []string{"git-lfs filter --skip"},
},
Upgradeables: upgradeables,
}
)

4
lfsapi/auth.go
View File

@ -49,10 +49,10 @@ func (c *Client) DoWithAuth(remote string, req *http.Request) (*http.Response, e
c.Endpoints.SetAccess(apiEndpoint.Url, newAccess)
}
if access == NoneAccess || creds != nil {
if creds != nil || (access == NoneAccess && len(req.Header.Get("Authorization")) == 0) {
tracerx.Printf("api: http response indicates %q authentication. Resubmitting...", newAccess)
req.Header.Del("Authorization")
if creds != nil {
req.Header.Del("Authorization")
credHelper.Reject(creds)
}
return c.DoWithAuth(remote, req)

9
lfsapi/client.go
View File

@ -97,12 +97,17 @@ func (c *Client) Close() error {
}
func (c *Client) extraHeadersFor(req *http.Request) http.Header {
extraHeaders := c.extraHeaders(req.URL)
if len(extraHeaders) == 0 {
return req.Header
}
copy := make(http.Header, len(req.Header))
for k, vs := range req.Header {
copy[k] = vs
}
for k, vs := range c.extraHeaders(req.URL) {
for k, vs := range extraHeaders {
for _, v := range vs {
copy[k] = append(copy[k], v)
}
@ -246,7 +251,7 @@ func (c *Client) httpClient(host string) *http.Client {
MaxIdleConnsPerHost: concurrentTransfers,
}
activityTimeout := 10
activityTimeout := 30
if v, ok := c.uc.Get("lfs", fmt.Sprintf("https://%v", host), "activitytimeout"); ok {
if i, err := strconv.Atoi(v); err == nil {
activityTimeout = i

17
lfsapi/creds.go
View File

@ -22,9 +22,11 @@ type credsConfig struct {
// See: https://git-scm.com/docs/gitcredentials#_requesting_credentials
// for more.
AskPass string `os:"GIT_ASKPASS" git:"core.askpass" os:"SSH_ASKPASS"`
// Helper is a string defining the credential helper that Git should use.
Helper string `git:"credential.helper"`
// Cached is a boolean determining whether or not to enable the
// credential cacher.
Cached bool `git:"lfs.cachecredentials"`
Cached bool
// SkipPrompt is a boolean determining whether or not to prompt the user
// for a password.
SkipPrompt bool `os:"GIT_TERMINAL_PROMPT"`
@ -42,7 +44,7 @@ func getCredentialHelper(cfg *config.Configuration) (CredentialHelper, error) {
}
var hs []CredentialHelper
if len(ccfg.AskPass) > 0 {
if len(ccfg.Helper) == 0 && len(ccfg.AskPass) > 0 {
hs = append(hs, &AskPassCredentialHelper{
Program: ccfg.AskPass,
})
@ -70,12 +72,15 @@ func getCredentialHelper(cfg *config.Configuration) (CredentialHelper, error) {
// getCredentialConfig parses a *credsConfig given the OS and Git
// configurations.
func getCredentialConfig(cfg *config.Configuration) (*credsConfig, error) {
var what credsConfig
what := &credsConfig{
Cached: cfg.Git.Bool("lfs.cachecredentials", true),
}
if err := cfg.Unmarshal(&what); err != nil {
if err := cfg.Unmarshal(what); err != nil {
return nil, err
}
return &what, nil
return what, nil
}
// CredentialHelpers is a []CredentialHelper that iterates through each
@ -118,7 +123,7 @@ func (h CredentialHelpers) Reject(what Creds) error {
}
// Approve implements CredentialHelper.Approve and approves the given Creds
// "what" amongst all knonw CredentialHelpers. If any `CredentialHelper`s
// "what" amongst all known CredentialHelpers. If any `CredentialHelper`s
// returned a non-nil error, no further `CredentialHelper`s are notified, so as
// to prevent inconsistent state.
func (h CredentialHelpers) Approve(what Creds) error {

4
lfsapi/lfsapi.go
View File

@ -64,9 +64,9 @@ func NewClient(osEnv Env, gitEnv Env) (*Client, error) {
gitEnv = make(TestEnv)
}
netrc, err := ParseNetrc(osEnv)
netrc, netrcfile, err := ParseNetrc(osEnv)
if err != nil {
return nil, err
return nil, errors.Wrap(err, fmt.Sprintf("bad netrc file %s", netrcfile))
}
httpsProxy, httpProxy, noProxy := getProxyServers(osEnv, gitEnv)

9
lfsapi/netrc.go
View File

@ -11,18 +11,19 @@ type NetrcFinder interface {
FindMachine(string) *netrc.Machine
}
func ParseNetrc(osEnv Env) (NetrcFinder, error) {
func ParseNetrc(osEnv Env) (NetrcFinder, string, error) {
home, _ := osEnv.Get("HOME")
if len(home) == 0 {
return &noFinder{}, nil
return &noFinder{}, "", nil
}
nrcfilename := filepath.Join(home, netrcBasename)
if _, err := os.Stat(nrcfilename); err != nil {
return &noFinder{}, nil
return &noFinder{}, nrcfilename, nil
}
return netrc.ParseFile(nrcfilename)
f, err := netrc.ParseFile(nrcfilename)
return f, nrcfilename, err
}
type noFinder struct{}

1
locking/lockable.go
View File

@ -39,7 +39,6 @@ func (c *Client) ensureLockablesLoaded() {
// Internal function to repopulate lockable patterns
// You must have locked the c.lockableMutex in the caller
func (c *Client) refreshLockablePatterns() {
paths := git.GetAttributePaths(c.LocalWorkingDir, c.LocalGitDir)
// Always make non-nil even if empty
c.lockablePatterns = make([]string, 0, len(paths))

29
locking/locks.go
View File

@ -144,22 +144,7 @@ func (c *Client) UnlockFile(path string, force bool) error {
return fmt.Errorf("Unable to get lock id: %v", err)
}
err = c.UnlockFileById(id, force)
if err != nil {
return err
}
abs, err := getAbsolutePath(path)
if err != nil {
return errors.Wrap(err, "make lockpath absolute")
}
// Make non-writeable if required
if c.SetLockableFilesReadOnly && c.IsFileLockable(path) {
return tools.SetFileWriteFlag(abs, false)
}
return nil
return c.UnlockFileById(id, force)
}
// UnlockFileById attempts to unlock a lock with a given id on the current remote
@ -181,6 +166,18 @@ func (c *Client) UnlockFileById(id string, force bool) error {
return fmt.Errorf("Error caching unlock information: %v", err)
}
if unlockRes.Lock != nil {
abs, err := getAbsolutePath(unlockRes.Lock.Path)
if err != nil {
return errors.Wrap(err, "make lockpath absolute")
}
// Make non-writeable if required
if c.SetLockableFilesReadOnly && c.IsFileLockable(unlockRes.Lock.Path) {
return tools.SetFileWriteFlag(abs, false)
}
}
return nil
}

2
rpm/SPECS/git-lfs.spec
View File

@ -1,5 +1,5 @@
Name: git-lfs
Version: 2.3.1
Version: 2.3.4
Release: 1%{?dist}
Summary: Git extension for versioning large files

41
script/build.go
View File

@ -19,8 +19,8 @@ import (
)
var (
BuildOS = flag.String("os", runtime.GOOS, "OS to target: darwin, freebsd, linux, windows")
BuildArch = flag.String("arch", "", "Arch to target: 386, amd64")
BuildOS = flag.String("os", "", "OS to target: darwin,freebsd,linux,windows")
BuildArch = flag.String("arch", "", "Arch to target: 386,amd64")
BuildAll = flag.Bool("all", false, "Builds all architectures")
BuildDwarf = flag.Bool("dwarf", false, "Includes DWARF tables in build artifacts")
BuildLdFlags = flag.String("ldflags", "", "-ldflags to pass to the compiler")
@ -64,21 +64,34 @@ func mainBuild() {
buildMatrix := make(map[string]Release)
errored := false
var platforms, arches []string
if len(*BuildOS) > 0 {
platforms = strings.Split(*BuildOS, ",")
}
if len(*BuildArch) > 0 {
arches = strings.Split(*BuildArch, ",")
}
if *BuildAll {
for _, buildos := range []string{"linux", "darwin", "freebsd", "windows"} {
for _, buildarch := range []string{"amd64", "386"} {
if err := build(buildos, buildarch, buildMatrix); err != nil {
errored = true
}
}
}
} else {
if err := build(*BuildOS, *BuildArch, buildMatrix); err != nil {
platforms = []string{"linux", "darwin", "freebsd", "windows"}
arches = []string{"amd64", "386"}
}
if len(platforms) < 1 || len(arches) < 1 {
if err := build("", "", buildMatrix); err != nil {
log.Fatalln(err)
}
return // skip build matrix stuff
}
for _, buildos := range platforms {
for _, buildarch := range arches {
err := build(strings.TrimSpace(buildos), strings.TrimSpace(buildarch), buildMatrix)
if err != nil {
errored = true
}
}
}
if errored {
os.Exit(1)
}
@ -141,7 +154,11 @@ func buildCommand(dir, buildos, buildarch string) error {
bin := filepath.Join(dir, "git-lfs")
if buildos == "windows" {
cmdOS := runtime.GOOS
if len(buildos) > 0 {
cmdOS = buildos
}
if cmdOS == "windows" {
bin = bin + ".exe"
}

3
test/test-askpass.sh
View File

@ -19,6 +19,7 @@ begin_test "askpass: push with GIT_ASKPASS"
# $password is defined from test/cmd/lfstest-gitserver.go (see: skipIfBadAuth)
export LFS_ASKPASS_USERNAME="user"
export LFS_ASKPASS_PASSWORD="pass"
git config "credential.helper" ""
GIT_ASKPASS="lfs-askpass" SSH_ASKPASS="dont-call-me" GIT_TRACE=1 GIT_CURL_VERBOSE=1 git push origin master 2>&1 | tee push.log
GITSERVER_USER="$(printf $GITSERVER | sed -e 's/http:\/\//http:\/\/user@/')"
@ -53,6 +54,7 @@ begin_test "askpass: push with core.askPass"
# $password is defined from test/cmd/lfstest-gitserver.go (see: skipIfBadAuth)
export LFS_ASKPASS_PASSWORD="pass"
git config "credential.helper" ""
git config "core.askPass" "lfs-askpass"
cat .git/config
SSH_ASKPASS="dont-call-me" GIT_TRACE=1 GIT_CURL_VERBOSE=1 git push origin master 2>&1 | tee push.log
@ -90,6 +92,7 @@ begin_test "askpass: push with SSH_ASKPASS"
# $password is defined from test/cmd/lfstest-gitserver.go (see: skipIfBadAuth)
export LFS_ASKPASS_USERNAME="user"
export LFS_ASKPASS_PASSWORD="pass"
git config "credential.helper" ""
SSH_ASKPASS="lfs-askpass" GIT_TRACE=1 GIT_CURL_VERBOSE=1 git push origin master 2>&1 | tee push.log
GITSERVER_USER="$(printf $GITSERVER | sed -e 's/http:\/\//http:\/\/user@/')"

34
test/test-clone.sh
View File

@ -55,12 +55,14 @@ begin_test "clone"
# check a few file sizes to make sure pulled
pushd "$newclonedir"
[ $(wc -c < "file1.dat") -eq 110 ]
[ $(wc -c < "file2.dat") -eq 75 ]
[ $(wc -c < "file3.dat") -eq 66 ]
assert_hooks "$(dot_git_dir)"
[ ! -e "lfs" ]
[ $(wc -c < "file1.dat") -eq 110 ]
[ $(wc -c < "file2.dat") -eq 75 ]
[ $(wc -c < "file3.dat") -eq 66 ]
assert_hooks "$(dot_git_dir)"
[ ! -e "lfs" ]
assert_clean_status
popd
# Now check clone with implied dir
rm -rf "$reponame"
git lfs clone "$GITSERVER/$reponame" 2>&1 | tee lfsclone.log
@ -71,12 +73,14 @@ begin_test "clone"
[ ! $(grep "error" lfsclone.log) ]
# clone location should be implied
[ -d "$reponame" ]
pushd "$reponame"
[ $(wc -c < "file1.dat") -eq 110 ]
[ $(wc -c < "file2.dat") -eq 75 ]
[ $(wc -c < "file3.dat") -eq 66 ]
assert_hooks "$(dot_git_dir)"
[ ! -e "lfs" ]
[ $(wc -c < "file1.dat") -eq 110 ]
[ $(wc -c < "file2.dat") -eq 75 ]
[ $(wc -c < "file3.dat") -eq 66 ]
assert_hooks "$(dot_git_dir)"
[ ! -e "lfs" ]
assert_clean_status
popd
)
@ -119,6 +123,7 @@ begin_test "cloneSSL"
newclonedir="testcloneSSL1"
git lfs clone "$SSLGITSERVER/$reponame" "$newclonedir" 2>&1 | tee lfsclone.log
assert_clean_status
grep "Cloning into" lfsclone.log
grep "Git LFS:" lfsclone.log
# should be no filter errors
@ -188,10 +193,11 @@ begin_test "clone ClientCert"
# check a few file sizes to make sure pulled
pushd "$newclonedir"
[ $(wc -c < "file1.dat") -eq 100 ]
[ $(wc -c < "file2.dat") -eq 75 ]
[ $(wc -c < "file3.dat") -eq 30 ]
assert_hooks "$(dot_git_dir)"
[ $(wc -c < "file1.dat") -eq 100 ]
[ $(wc -c < "file2.dat") -eq 75 ]
[ $(wc -c < "file3.dat") -eq 30 ]
assert_hooks "$(dot_git_dir)"
assert_clean_status
popd

25
test/test-install.sh
View File

@ -29,15 +29,10 @@ begin_test "install with old (non-upgradeable) settings"
git config --global filter.lfs.smudge "git-lfs smudge --something %f"
git config --global filter.lfs.clean "git-lfs clean --something %f"
set +e
git lfs install 2> install.log
res=$?
set -e
git lfs install | tee install.log
[ "${PIPESTATUS[0]}" = 0 ]
[ "$res" = 2 ]
cat install.log
grep -E "(clean|smudge) attribute should be" install.log
grep -E "(clean|smudge)\" attribute should be" install.log
[ `grep -c "(MISSING)" install.log` = "0" ]
[ "git-lfs smudge --something %f" = "$(git config --global filter.lfs.smudge)" ]
@ -194,7 +189,7 @@ begin_test "install --skip-smudge"
[ "git-lfs smudge --skip -- %f" = "$(git config --global filter.lfs.smudge)" ]
[ "git-lfs filter-process --skip" = "$(git config --global filter.lfs.process)" ]
git lfs install --force
git lfs install
[ "git-lfs clean -- %f" = "$(git config --global filter.lfs.clean)" ]
[ "git-lfs smudge -- %f" = "$(git config --global filter.lfs.smudge)" ]
[ "git-lfs filter-process" = "$(git config --global filter.lfs.process)" ]
@ -283,3 +278,15 @@ begin_test "install in repo without changing hooks"
[ "git-lfs filter-process" = "$(git config filter.lfs.process)" ]
)
end_test
begin_test "can install when multiple global values registered"
(
set -e
git config --global filter.lfs.smudge "git-lfs smudge --something %f"
git config --global --add filter.lfs.smudge "git-lfs smudge --something-else %f"
git lfs install --force
)
end_test

51
test/test-pull.sh
View File

@ -19,33 +19,42 @@ begin_test "pull"
contents_oid=$(calc_oid "$contents")
contents2="A"
contents2_oid=$(calc_oid "$contents2")
contents3="dir"
contents3_oid=$(calc_oid "$contents3")
mkdir dir
echo "*.log" > .gitignore
printf "$contents" > a.dat
printf "$contents2" > á.dat
git add a.dat á.dat .gitattributes
printf "$contents3" > dir/dir.dat
git add .
git commit -m "add files" 2>&1 | tee commit.log
grep "master (root-commit)" commit.log
grep "3 files changed" commit.log
grep "5 files changed" commit.log
grep "create mode 100644 a.dat" commit.log
grep "create mode 100644 .gitattributes" commit.log
ls -al
[ "a" = "$(cat a.dat)" ]
[ "A" = "$(cat "á.dat")" ]
[ "dir" = "$(cat "dir/dir.dat")" ]
assert_pointer "master" "a.dat" "$contents_oid" 1
assert_pointer "master" "á.dat" "$contents2_oid" 1
assert_pointer "master" "dir/dir.dat" "$contents3_oid" 3
refute_server_object "$reponame" "$contents_oid"
refute_server_object "$reponame" "$contents2_oid"
refute_server_object "$reponame" "$contents33oid"
echo "initial push"
git push origin master 2>&1 | tee push.log
grep "(2 of 2 files)" push.log
grep "(3 of 3 files)" push.log
grep "master -> master" push.log
assert_server_object "$reponame" "$contents_oid"
assert_server_object "$reponame" "$contents2_oid"
assert_server_object "$reponame" "$contents3_oid"
# change to the clone's working directory
cd ../clone
@ -58,11 +67,12 @@ begin_test "pull"
assert_local_object "$contents_oid" 1
assert_local_object "$contents2_oid" 1
assert_clean_status
echo "lfs pull"
rm a.dat á.dat
rm -r a.dat á.dat dir # removing files makes the status dirty
rm -rf .git/lfs/objects
git lfs pull 2>&1 | grep "(2 of 2 files)"
git lfs pull 2>&1 | grep "(3 of 3 files)"
ls -al
[ "a" = "$(cat a.dat)" ]
[ "A" = "$(cat "á.dat")" ]
@ -70,41 +80,68 @@ begin_test "pull"
assert_local_object "$contents2_oid" 1
echo "lfs pull with remote"
rm a.dat á.dat
rm -r a.dat á.dat dir
rm -rf .git/lfs/objects
git lfs pull origin 2>&1 | grep "(2 of 2 files)"
git lfs pull origin 2>&1 | grep "(3 of 3 files)"
[ "a" = "$(cat a.dat)" ]
[ "A" = "$(cat "á.dat")" ]
assert_local_object "$contents_oid" 1
assert_local_object "$contents2_oid" 1
assert_clean_status
echo "lfs pull with local storage"
rm a.dat á.dat
git lfs pull
[ "a" = "$(cat a.dat)" ]
[ "A" = "$(cat "á.dat")" ]
assert_clean_status
echo "lfs pull with include/exclude filters in gitconfig"
rm -rf .git/lfs/objects
git config "lfs.fetchinclude" "a*"
git lfs pull
assert_local_object "$contents_oid" 1
assert_clean_status
rm -rf .git/lfs/objects
git config --unset "lfs.fetchinclude"
git config "lfs.fetchexclude" "a*"
git lfs pull
refute_local_object "$contents_oid"
assert_clean_status
echo "lfs pull with include/exclude filters in command line"
git config --unset "lfs.fetchexclude"
rm -rf .git/lfs/objects
git lfs pull --include="a*"
assert_local_object "$contents_oid" 1
assert_clean_status
rm -rf .git/lfs/objects
git lfs pull --exclude="a*"
refute_local_object "$contents_oid"
assert_clean_status
echo "resetting to test status"
git reset --hard
assert_clean_status
echo "lfs pull clean status"
git lfs pull
assert_clean_status
echo "lfs pull with -I"
git lfs pull -I "*.dat"
assert_clean_status
echo "lfs pull in subdir"
cd dir
git lfs pull
assert_clean_status
echo "lfs pull in subdir with -I"
git lfs pull -I "*.dat"
assert_clean_status
)
end_test

5
test/test-track.sh
View File

@ -93,7 +93,9 @@ begin_test "track directory"
cd dir
git init
git lfs track "foo bar/*"
git lfs track "foo bar\\*" | tee track.txt
[ "foo[[:space:]]bar/* filter=lfs diff=lfs merge=lfs -text" = "$(cat .gitattributes)" ]
[ "Tracking \"foo bar/*\"" = "$(cat track.txt)" ]
mkdir "foo bar"
echo "a" > "foo bar/a"
@ -104,6 +106,7 @@ begin_test "track directory"
assert_pointer "master" "foo bar/a" "87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7" 2
assert_pointer "master" "foo bar/b" "0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f" 2
)
end_test
begin_test "track without trailing linebreak"
(

7
test/test-unlock.sh
View File

@ -34,7 +34,7 @@ begin_test "unlocking a file makes it readonly"
)
end_test
begin_test "unlocking a file makes ignores readonly"
begin_test "unlocking a file ignores readonly"
(
set -e
@ -99,15 +99,16 @@ begin_test "unlocking a lock by id"
set -e
reponame="unlock_by_id"
setup_remote_repo_with_file "unlock_by_id" "d.dat"
setup_remote_repo_with_file "$reponame" "d.dat"
git lfs lock --json "d.dat" | tee lock.log
assert_file_writeable d.dat
id=$(assert_lock lock.log d.dat)
assert_server_lock "$reponame" "$id"
git lfs unlock --id="$id"
refute_server_lock "$reponame" "$id"
refute_file_writeable d.dat
)
end_test

9
test/testhelpers.sh
View File

@ -238,6 +238,14 @@ assert_hooks() {
[ -x "$git_root/hooks/pre-push" ]
}
assert_clean_status() {
status="$(git status)"
echo "$status" | grep "working tree clean" || {
echo $status
git lfs status
}
}
# pointer returns a string Git LFS pointer file.
#
# $ pointer abc-some-oid 123 <version>
@ -340,7 +348,6 @@ clone_repo_url() {
echo "$out"
}
# clone_repo_ssl clones a repository from the test Git server to the subdirectory
# $dir under $TRASHDIR, using the SSL endpoint.
# setup_remote_repo() needs to be run first. Output is written to clone_ssl.log.

129
tools/filetools.go
View File

@ -10,8 +10,11 @@ import (
"os"
"path"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"github.com/git-lfs/git-lfs/filepathfilter"
)
@ -142,14 +145,8 @@ type FastWalkCallback func(parentDir string, info os.FileInfo, err error)
//
// rootDir - Absolute path to the top of the repository working directory
func FastWalkGitRepo(rootDir string, cb FastWalkCallback) {
// Ignore all git metadata including subrepos
excludePaths := []filepathfilter.Pattern{
filepathfilter.NewPattern(".git"),
filepathfilter.NewPattern(filepath.Join("**", ".git")),
}
fileCh := fastWalkWithExcludeFiles(rootDir, ".gitignore", excludePaths)
for file := range fileCh {
walker := fastWalkWithExcludeFiles(rootDir, ".gitignore")
for file := range walker.ch {
cb(file.ParentDir, file.Info, file.Err)
}
}
@ -163,53 +160,71 @@ type fastWalkInfo struct {
Err error
}
type fastWalker struct {
rootDir string
excludeFilename string
ch chan fastWalkInfo
limit int32
cur *int32
wg *sync.WaitGroup
}
// fastWalkWithExcludeFiles walks the contents of a dir, respecting
// include/exclude patterns and also loading new exlude patterns from files
// named excludeFilename in directories walked
//
// rootDir - Absolute path to the top of the repository working directory
func fastWalkWithExcludeFiles(rootDir, excludeFilename string,
excludePaths []filepathfilter.Pattern) <-chan fastWalkInfo {
fiChan := make(chan fastWalkInfo, 256)
go fastWalkFromRoot(rootDir, excludeFilename, excludePaths, fiChan)
return fiChan
}
// rootDir - Absolute path to the top of the repository working directory
func fastWalkFromRoot(rootDir string, excludeFilename string,
excludePaths []filepathfilter.Pattern, fiChan chan<- fastWalkInfo) {
dirFi, err := os.Stat(rootDir)
if err != nil {
fiChan <- fastWalkInfo{Err: err}
return
func fastWalkWithExcludeFiles(rootDir, excludeFilename string) *fastWalker {
excludePaths := []filepathfilter.Pattern{
filepathfilter.NewPattern(".git"),
filepathfilter.NewPattern(filepath.Join("**", ".git")),
}
// This waitgroup will be incremented for each nested goroutine
var waitg sync.WaitGroup
fastWalkFileOrDir(true, rootDir, "", dirFi, excludeFilename, excludePaths, fiChan, &waitg)
waitg.Wait()
close(fiChan)
limit, _ := strconv.Atoi(os.Getenv("LFS_FASTWALK_LIMIT"))
if limit < 1 {
limit = runtime.GOMAXPROCS(-1) * 20
}
c := int32(0)
w := &fastWalker{
rootDir: rootDir,
excludeFilename: excludeFilename,
limit: int32(limit),
cur: &c,
ch: make(chan fastWalkInfo, 256),
wg: &sync.WaitGroup{},
}
go func() {
dirFi, err := os.Stat(w.rootDir)
if err != nil {
w.ch <- fastWalkInfo{Err: err}
return
}
w.Walk(true, "", dirFi, excludePaths)
w.Wait()
}()
return w
}
// fastWalkFileOrDir is the main recursive implementation of fast walk
// Walk is the main recursive implementation of fast walk.
// Sends the file/dir and any contents to the channel so long as it passes the
// include/exclude filter. If a dir, parses any excludeFilename found and updates
// the excludePaths with its content before (parallel) recursing into contents
// Also splits large directories into multiple goroutines.
// Increments waitg.Add(1) for each new goroutine launched internally
//
// rootDir - Absolute path to the top of the repository working directory
// workDir - Relative path inside the repository
func fastWalkFileOrDir(isRoot bool, rootDir, workDir string, itemFi os.FileInfo, excludeFilename string,
excludePaths []filepathfilter.Pattern, fiChan chan<- fastWalkInfo, waitg *sync.WaitGroup) {
func (w *fastWalker) Walk(isRoot bool, workDir string, itemFi os.FileInfo,
excludePaths []filepathfilter.Pattern) {
var fullPath string // Absolute path to the current file or dir
var parentWorkDir string // Absolute path to the workDir inside the repository
if isRoot {
fullPath = rootDir
fullPath = w.rootDir
} else {
parentWorkDir = filepath.Join(rootDir, workDir)
parentWorkDir = filepath.Join(w.rootDir, workDir)
fullPath = filepath.Join(parentWorkDir, itemFi.Name())
}
@ -218,7 +233,7 @@ func fastWalkFileOrDir(isRoot bool, rootDir, workDir string, itemFi os.FileInfo,
return
}
fiChan <- fastWalkInfo{ParentDir: parentWorkDir, Info: itemFi}
w.ch <- fastWalkInfo{ParentDir: parentWorkDir, Info: itemFi}
if !itemFi.IsDir() {
// Nothing more to do if this is not a dir
@ -230,12 +245,12 @@ func fastWalkFileOrDir(isRoot bool, rootDir, workDir string, itemFi os.FileInfo,
childWorkDir = filepath.Join(workDir, itemFi.Name())
}
if len(excludeFilename) > 0 {
possibleExcludeFile := filepath.Join(fullPath, excludeFilename)
if len(w.excludeFilename) > 0 {
possibleExcludeFile := filepath.Join(fullPath, w.excludeFilename)
var err error
excludePaths, err = loadExcludeFilename(possibleExcludeFile, childWorkDir, excludePaths)
if err != nil {
fiChan <- fastWalkInfo{Err: err}
w.ch <- fastWalkInfo{Err: err}
}
}
@ -245,30 +260,48 @@ func fastWalkFileOrDir(isRoot bool, rootDir, workDir string, itemFi os.FileInfo,
// filepath.Walk as a bonus.
df, err := os.Open(fullPath)
if err != nil {
fiChan <- fastWalkInfo{Err: err}
w.ch <- fastWalkInfo{Err: err}
return
}
defer df.Close()
// The number of items in a dir we process in each goroutine
jobSize := 100
for children, err := df.Readdir(jobSize); err == nil; children, err = df.Readdir(jobSize) {
// Parallelise all dirs, and chop large dirs into batches
waitg.Add(1)
go func(subitems []os.FileInfo) {
w.walk(children, func(subitems []os.FileInfo) {
for _, childFi := range subitems {
fastWalkFileOrDir(false, rootDir, childWorkDir, childFi, excludeFilename, excludePaths, fiChan, waitg)
w.Walk(false, childWorkDir, childFi, excludePaths)
}
waitg.Done()
}(children)
})
}
}
df.Close()
if err != nil && err != io.EOF {
fiChan <- fastWalkInfo{Err: err}
w.ch <- fastWalkInfo{Err: err}
}
}
func (w *fastWalker) walk(children []os.FileInfo, fn func([]os.FileInfo)) {
cur := atomic.AddInt32(w.cur, 1)
if cur > w.limit {
fn(children)
atomic.AddInt32(w.cur, -1)
return
}
w.wg.Add(1)
go func() {
fn(children)
w.wg.Done()
atomic.AddInt32(w.cur, -1)
}()
}
func (w *fastWalker) Wait() {
w.wg.Wait()
close(w.ch)
}
// loadExcludeFilename reads the given file in gitignore format and returns a
// revised array of exclude paths if there are any changes.
// If any changes are made a copy of the array is taken so the original is not

9
tools/filetools_test.go
View File

@ -15,13 +15,11 @@ import (
func TestCleanPathsCleansPaths(t *testing.T) {
cleaned := CleanPaths("/foo/bar/,/foo/bar/baz", ",")
assert.Equal(t, []string{"/foo/bar", "/foo/bar/baz"}, cleaned)
}
func TestCleanPathsReturnsNoResultsWhenGivenNoPaths(t *testing.T) {
cleaned := CleanPaths("", ",")
assert.Empty(t, cleaned)
}
@ -35,15 +33,14 @@ func TestFastWalkBasic(t *testing.T) {
expectedEntries := createFastWalkInputData(10, 160)
fchan := fastWalkWithExcludeFiles(expectedEntries[0], "", nil)
gotEntries, gotErrors := collectFastWalkResults(fchan)
walker := fastWalkWithExcludeFiles(expectedEntries[0], "")
gotEntries, gotErrors := collectFastWalkResults(walker.ch)
assert.Empty(t, gotErrors)
sort.Strings(expectedEntries)
sort.Strings(gotEntries)
assert.Equal(t, expectedEntries, gotEntries)
}
func BenchmarkFastWalkGitRepoChannels(b *testing.B) {
@ -229,7 +226,6 @@ func getFileMode(filename string) os.FileMode {
}
func TestSetWriteFlag(t *testing.T) {
f, err := ioutil.TempFile("", "lfstestwriteflag")
assert.Nil(t, err)
filename := f.Name()
@ -272,5 +268,4 @@ func TestSetWriteFlag(t *testing.T) {
// should only add back user write
assert.EqualValues(t, 0640, getFileMode(filename))
}
}

4
versioninfo.json
View File

@ -4,7 +4,7 @@
"FileVersion": {
"Major": 2,
"Minor": 3,
"Patch": 1,
"Patch": 4,
"Build": 0
}
},
@ -13,6 +13,6 @@
"FileDescription": "Git LFS",
"LegalCopyright": "GitHub, Inc. and Git LFS contributors",
"ProductName": "Git Large File Storage (LFS)",
"ProductVersion": "2.3.0"
"ProductVersion": "2.3.4"
}
}