2016-11-08 09:58:07 +00:00
|
|
|
package tools
|
2016-05-31 16:21:36 +00:00
|
|
|
|
|
|
|
import (
|
2016-10-31 15:39:52 +00:00
|
|
|
"fmt"
|
|
|
|
"io/ioutil"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2016-12-19 12:39:51 +00:00
|
|
|
"runtime"
|
2016-10-31 15:39:52 +00:00
|
|
|
"sort"
|
2016-05-31 16:21:36 +00:00
|
|
|
"testing"
|
|
|
|
|
2016-11-15 17:01:18 +00:00
|
|
|
"github.com/git-lfs/git-lfs/subprocess"
|
2016-05-31 16:21:36 +00:00
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
)
|
|
|
|
|
|
|
|
func TestCleanPathsCleansPaths(t *testing.T) {
|
2016-11-08 09:58:07 +00:00
|
|
|
cleaned := CleanPaths("/foo/bar/,/foo/bar/baz", ",")
|
2016-05-31 16:21:36 +00:00
|
|
|
|
|
|
|
assert.Equal(t, []string{"/foo/bar", "/foo/bar/baz"}, cleaned)
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestCleanPathsReturnsNoResultsWhenGivenNoPaths(t *testing.T) {
|
2016-11-08 09:58:07 +00:00
|
|
|
cleaned := CleanPaths("", ",")
|
2016-05-31 16:21:36 +00:00
|
|
|
|
|
|
|
assert.Empty(t, cleaned)
|
|
|
|
}
|
2016-10-31 12:45:12 +00:00
|
|
|
|
2016-10-31 15:39:52 +00:00
|
|
|
func TestFastWalkBasic(t *testing.T) {
|
|
|
|
rootDir, err := ioutil.TempDir(os.TempDir(), "GitLfsTestFastWalkBasic")
|
|
|
|
if err != nil {
|
|
|
|
assert.FailNow(t, "Unable to get temp dir: %v", err)
|
|
|
|
}
|
2016-11-01 14:36:03 +00:00
|
|
|
defer os.RemoveAll(rootDir)
|
|
|
|
os.Chdir(rootDir)
|
|
|
|
|
|
|
|
expectedEntries := createFastWalkInputData(10, 160)
|
|
|
|
|
2016-11-29 15:14:35 +00:00
|
|
|
fchan := fastWalkWithExcludeFiles(expectedEntries[0], "", nil)
|
|
|
|
gotEntries, gotErrors := collectFastWalkResults(fchan)
|
2016-11-01 14:36:03 +00:00
|
|
|
|
|
|
|
assert.Empty(t, gotErrors)
|
|
|
|
|
|
|
|
sort.Strings(expectedEntries)
|
|
|
|
sort.Strings(gotEntries)
|
|
|
|
assert.Equal(t, expectedEntries, gotEntries)
|
2016-10-31 15:39:52 +00:00
|
|
|
|
2016-11-01 14:36:03 +00:00
|
|
|
}
|
|
|
|
|
2016-11-22 22:28:16 +00:00
|
|
|
func BenchmarkFastWalkGitRepoChannels(b *testing.B) {
|
|
|
|
rootDir, err := ioutil.TempDir(os.TempDir(), "GitLfsBenchFastWalkGitRepoChannels")
|
|
|
|
if err != nil {
|
|
|
|
assert.FailNow(b, "Unable to get temp dir: %v", err)
|
|
|
|
}
|
|
|
|
defer os.RemoveAll(rootDir)
|
|
|
|
os.Chdir(rootDir)
|
|
|
|
entries := createFastWalkInputData(1000, 5000)
|
|
|
|
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
|
|
var files, errors int
|
2016-11-23 20:28:11 +00:00
|
|
|
FastWalkGitRepo(entries[0], func(parent string, info os.FileInfo, err error) {
|
|
|
|
if err != nil {
|
2016-11-22 22:28:16 +00:00
|
|
|
errors++
|
2016-11-23 20:28:11 +00:00
|
|
|
} else {
|
|
|
|
files++
|
2016-11-22 22:28:16 +00:00
|
|
|
}
|
2016-11-23 20:28:11 +00:00
|
|
|
})
|
2016-11-22 22:28:16 +00:00
|
|
|
b.Logf("files: %d, errors: %d", files, errors)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func BenchmarkFastWalkGitRepoCallback(b *testing.B) {
|
|
|
|
rootDir, err := ioutil.TempDir(os.TempDir(), "GitLfsBenchFastWalkGitRepoCallback")
|
|
|
|
if err != nil {
|
|
|
|
assert.FailNow(b, "Unable to get temp dir: %v", err)
|
|
|
|
}
|
|
|
|
defer os.RemoveAll(rootDir)
|
|
|
|
os.Chdir(rootDir)
|
|
|
|
entries := createFastWalkInputData(1000, 5000)
|
|
|
|
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
|
|
var files, errors int
|
|
|
|
FastWalkGitRepo(entries[0], func(parentDir string, info os.FileInfo, err error) {
|
|
|
|
if err != nil {
|
|
|
|
errors++
|
|
|
|
} else {
|
|
|
|
files++
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
b.Logf("files: %d, errors: %d", files, errors)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-11-01 14:36:03 +00:00
|
|
|
func TestFastWalkGitRepo(t *testing.T) {
|
|
|
|
rootDir, err := ioutil.TempDir(os.TempDir(), "GitLfsTestFastWalkGitRepo")
|
|
|
|
if err != nil {
|
|
|
|
assert.FailNow(t, "Unable to get temp dir: %v", err)
|
|
|
|
}
|
2016-10-31 15:39:52 +00:00
|
|
|
defer os.RemoveAll(rootDir)
|
|
|
|
os.Chdir(rootDir)
|
2016-11-01 14:36:03 +00:00
|
|
|
|
|
|
|
expectedEntries := createFastWalkInputData(3, 3)
|
|
|
|
|
|
|
|
mainDir := expectedEntries[0]
|
|
|
|
|
|
|
|
// Set up a git repo and add some ignored files / dirs
|
|
|
|
subprocess.SimpleExec("git", "init", mainDir)
|
|
|
|
ignored := []string{
|
|
|
|
"filethatweignore.ign",
|
|
|
|
"foldercontainingignored",
|
|
|
|
"foldercontainingignored/notthisone.ign",
|
|
|
|
"ignoredfolder",
|
|
|
|
"ignoredfolder/file1.txt",
|
|
|
|
"ignoredfolder/file2.txt",
|
|
|
|
"ignoredfrominside",
|
|
|
|
"ignoredfrominside/thisisok.txt",
|
|
|
|
"ignoredfrominside/thisisnot.txt",
|
|
|
|
"ignoredfrominside/thisone",
|
|
|
|
"ignoredfrominside/thisone/file1.txt",
|
|
|
|
}
|
|
|
|
for _, f := range ignored {
|
|
|
|
fullPath := filepath.Join(mainDir, f)
|
|
|
|
if len(filepath.Ext(f)) > 0 {
|
|
|
|
ioutil.WriteFile(fullPath, []byte("TEST"), 0644)
|
|
|
|
} else {
|
|
|
|
os.MkdirAll(fullPath, 0755)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// write root .gitignore
|
|
|
|
rootGitIgnore := `
|
|
|
|
# ignore *.ign everywhere
|
|
|
|
*.ign
|
|
|
|
# ignore folder
|
|
|
|
ignoredfolder
|
|
|
|
`
|
|
|
|
ioutil.WriteFile(filepath.Join(mainDir, ".gitignore"), []byte(rootGitIgnore), 0644)
|
|
|
|
// Subfolder ignore; folder will show up but but subfolder 'thisone' won't
|
|
|
|
subFolderIgnore := `
|
|
|
|
thisone
|
|
|
|
thisisnot.txt
|
|
|
|
`
|
|
|
|
ioutil.WriteFile(filepath.Join(mainDir, "ignoredfrominside", ".gitignore"), []byte(subFolderIgnore), 0644)
|
|
|
|
|
|
|
|
// This dir will be walked but content won't be
|
|
|
|
expectedEntries = append(expectedEntries, filepath.Join(mainDir, "foldercontainingignored"))
|
|
|
|
// This dir will be walked and some of its content but has its own gitignore
|
|
|
|
expectedEntries = append(expectedEntries, filepath.Join(mainDir, "ignoredfrominside"))
|
|
|
|
expectedEntries = append(expectedEntries, filepath.Join(mainDir, "ignoredfrominside", "thisisok.txt"))
|
|
|
|
// Also gitignores
|
|
|
|
expectedEntries = append(expectedEntries, filepath.Join(mainDir, ".gitignore"))
|
|
|
|
expectedEntries = append(expectedEntries, filepath.Join(mainDir, "ignoredfrominside", ".gitignore"))
|
|
|
|
// nothing else should be there
|
|
|
|
|
2016-11-23 20:28:11 +00:00
|
|
|
gotEntries := make([]string, 0, 1000)
|
|
|
|
gotErrors := make([]error, 0, 5)
|
|
|
|
FastWalkGitRepo(mainDir, func(parent string, info os.FileInfo, err error) {
|
|
|
|
if err != nil {
|
|
|
|
gotErrors = append(gotErrors, err)
|
|
|
|
} else {
|
|
|
|
gotEntries = append(gotEntries, filepath.Join(parent, info.Name()))
|
|
|
|
}
|
|
|
|
})
|
2016-11-01 14:36:03 +00:00
|
|
|
|
|
|
|
assert.Empty(t, gotErrors)
|
|
|
|
|
|
|
|
sort.Strings(expectedEntries)
|
|
|
|
sort.Strings(gotEntries)
|
|
|
|
assert.Equal(t, expectedEntries, gotEntries)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
// Make test data - ensure you've Chdir'ed into a temp dir first
|
|
|
|
// Returns list of files/dirs that are created
|
|
|
|
// First entry is the parent dir of all others
|
|
|
|
func createFastWalkInputData(smallFolder, largeFolder int) []string {
|
2016-10-31 15:39:52 +00:00
|
|
|
dirs := []string{
|
|
|
|
"testroot",
|
|
|
|
"testroot/folder1",
|
|
|
|
"testroot/folder2",
|
|
|
|
"testroot/folder2/subfolder1",
|
|
|
|
"testroot/folder2/subfolder2",
|
|
|
|
"testroot/folder2/subfolder3",
|
|
|
|
"testroot/folder2/subfolder4",
|
|
|
|
"testroot/folder2/subfolder4/subsub",
|
|
|
|
}
|
|
|
|
expectedEntries := make([]string, 0, 250)
|
|
|
|
|
|
|
|
for i, dir := range dirs {
|
|
|
|
os.MkdirAll(dir, 0755)
|
2016-11-01 14:36:03 +00:00
|
|
|
numFiles := smallFolder
|
2016-11-01 14:41:33 +00:00
|
|
|
expectedEntries = append(expectedEntries, filepath.Clean(dir))
|
2016-10-31 15:39:52 +00:00
|
|
|
if i >= 3 && i <= 5 {
|
|
|
|
// Bulk test to ensure works with > 1 batch
|
2016-11-01 14:36:03 +00:00
|
|
|
numFiles = largeFolder
|
2016-10-31 15:39:52 +00:00
|
|
|
}
|
|
|
|
for f := 0; f < numFiles; f++ {
|
|
|
|
filename := filepath.Join(dir, fmt.Sprintf("file%d.txt", f))
|
2016-11-01 10:23:30 +00:00
|
|
|
ioutil.WriteFile(filename, []byte("TEST"), 0644)
|
2016-11-01 14:37:58 +00:00
|
|
|
expectedEntries = append(expectedEntries, filepath.Clean(filename))
|
2016-10-31 15:39:52 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-11-01 14:36:03 +00:00
|
|
|
return expectedEntries
|
2016-10-31 15:39:52 +00:00
|
|
|
}
|
|
|
|
|
2016-11-29 15:14:35 +00:00
|
|
|
func collectFastWalkResults(fchan <-chan fastWalkInfo) ([]string, []error) {
|
2016-10-31 15:39:52 +00:00
|
|
|
gotEntries := make([]string, 0, 1000)
|
|
|
|
gotErrors := make([]error, 0, 5)
|
2016-11-29 15:14:35 +00:00
|
|
|
for o := range fchan {
|
|
|
|
if o.Err != nil {
|
|
|
|
gotErrors = append(gotErrors, o.Err)
|
|
|
|
} else {
|
2016-10-31 15:39:52 +00:00
|
|
|
gotEntries = append(gotEntries, filepath.Join(o.ParentDir, o.Info.Name()))
|
|
|
|
}
|
2016-11-29 15:14:35 +00:00
|
|
|
}
|
2016-10-31 15:39:52 +00:00
|
|
|
|
|
|
|
return gotEntries, gotErrors
|
|
|
|
}
|
2016-12-19 12:39:51 +00:00
|
|
|
|
|
|
|
func getFileMode(filename string) os.FileMode {
|
|
|
|
s, err := os.Stat(filename)
|
|
|
|
if err != nil {
|
|
|
|
return 0000
|
|
|
|
}
|
|
|
|
return s.Mode()
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestSetWriteFlag(t *testing.T) {
|
|
|
|
|
|
|
|
f, err := ioutil.TempFile("", "lfstestwriteflag")
|
|
|
|
assert.Nil(t, err)
|
|
|
|
filename := f.Name()
|
|
|
|
defer os.Remove(filename)
|
|
|
|
f.Close()
|
|
|
|
// Set up with read/write bit for all but no execute
|
|
|
|
assert.Nil(t, os.Chmod(filename, 0666))
|
|
|
|
|
|
|
|
assert.Nil(t, SetFileWriteFlag(filename, false))
|
|
|
|
// should turn off all write
|
|
|
|
assert.EqualValues(t, 0444, getFileMode(filename))
|
|
|
|
assert.Nil(t, SetFileWriteFlag(filename, true))
|
|
|
|
// should only add back user write (on Mac/Linux)
|
|
|
|
if runtime.GOOS == "windows" {
|
|
|
|
assert.EqualValues(t, 0666, getFileMode(filename))
|
|
|
|
} else {
|
|
|
|
assert.EqualValues(t, 0644, getFileMode(filename))
|
|
|
|
}
|
|
|
|
|
|
|
|
// Can't run selective UGO tests on Windows as doesn't support separate roles
|
|
|
|
// Also Golang only supports read/write but not execute on Windows
|
|
|
|
if runtime.GOOS != "windows" {
|
|
|
|
// Set up with read/write/execute bit for all but no execute
|
|
|
|
assert.Nil(t, os.Chmod(filename, 0777))
|
|
|
|
assert.Nil(t, SetFileWriteFlag(filename, false))
|
|
|
|
// should turn off all write but not execute
|
|
|
|
assert.EqualValues(t, 0555, getFileMode(filename))
|
|
|
|
assert.Nil(t, SetFileWriteFlag(filename, true))
|
|
|
|
// should only add back user write (on Mac/Linux)
|
|
|
|
if runtime.GOOS == "windows" {
|
|
|
|
assert.EqualValues(t, 0777, getFileMode(filename))
|
|
|
|
} else {
|
|
|
|
assert.EqualValues(t, 0755, getFileMode(filename))
|
|
|
|
}
|
|
|
|
|
|
|
|
assert.Nil(t, os.Chmod(filename, 0440))
|
|
|
|
assert.Nil(t, SetFileWriteFlag(filename, false))
|
|
|
|
assert.EqualValues(t, 0440, getFileMode(filename))
|
|
|
|
assert.Nil(t, SetFileWriteFlag(filename, true))
|
|
|
|
// should only add back user write
|
|
|
|
assert.EqualValues(t, 0640, getFileMode(filename))
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|