2017-10-24 21:22:13 +00:00
|
|
|
package fs
|
2016-05-13 16:38:06 +00:00
|
|
|
|
|
|
|
import (
|
2018-07-09 15:28:23 +00:00
|
|
|
"bufio"
|
2018-06-13 14:56:46 +00:00
|
|
|
"bytes"
|
2021-09-27 12:33:22 +00:00
|
|
|
"crypto/sha256"
|
|
|
|
"encoding/hex"
|
2021-12-14 16:05:03 +00:00
|
|
|
"errors"
|
2016-05-13 16:38:06 +00:00
|
|
|
"io/ioutil"
|
2017-10-24 22:21:15 +00:00
|
|
|
"os"
|
2016-05-13 16:38:06 +00:00
|
|
|
"path/filepath"
|
2017-10-25 15:49:46 +00:00
|
|
|
"regexp"
|
2018-06-13 14:56:46 +00:00
|
|
|
"strconv"
|
2016-05-13 16:38:06 +00:00
|
|
|
"strings"
|
2017-10-24 22:21:15 +00:00
|
|
|
"sync"
|
2016-05-13 16:38:06 +00:00
|
|
|
|
2021-09-01 19:41:10 +00:00
|
|
|
"github.com/git-lfs/git-lfs/v3/tools"
|
2021-12-14 16:05:03 +00:00
|
|
|
"github.com/git-lfs/git-lfs/v3/tr"
|
2018-07-09 15:28:23 +00:00
|
|
|
"github.com/rubyist/tracerx"
|
2016-05-13 16:38:06 +00:00
|
|
|
)
|
|
|
|
|
2021-09-27 12:33:22 +00:00
|
|
|
var (
|
|
|
|
oidRE = regexp.MustCompile(`\A[[:alnum:]]{64}`)
|
|
|
|
EmptyObjectSHA256 = hex.EncodeToString(sha256.New().Sum(nil))
|
|
|
|
)
|
2017-10-25 15:49:46 +00:00
|
|
|
|
2018-07-09 16:23:27 +00:00
|
|
|
// Environment is a copy of a subset of the interface
|
|
|
|
// github.com/git-lfs/git-lfs/config.Environment.
|
|
|
|
//
|
|
|
|
// For more information, see config/environment.go.
|
|
|
|
type Environment interface {
|
|
|
|
Get(key string) (val string, ok bool)
|
|
|
|
}
|
|
|
|
|
2017-10-25 15:49:46 +00:00
|
|
|
// Object represents a locally stored LFS object.
|
|
|
|
type Object struct {
|
|
|
|
Oid string
|
|
|
|
Size int64
|
|
|
|
}
|
|
|
|
|
2017-10-24 21:22:13 +00:00
|
|
|
type Filesystem struct {
|
2018-07-09 15:28:23 +00:00
|
|
|
GitStorageDir string // parent of objects/lfs (may be same as GitDir but may not)
|
|
|
|
LFSStorageDir string // parent of lfs objects and tmp dirs. Default: ".git/lfs"
|
|
|
|
ReferenceDirs []string // alternative local media dirs (relative to clone reference repo)
|
2017-10-25 01:16:14 +00:00
|
|
|
lfsobjdir string
|
2017-10-25 00:59:36 +00:00
|
|
|
tmpdir string
|
2017-10-24 22:21:15 +00:00
|
|
|
logdir string
|
2018-12-04 22:25:02 +00:00
|
|
|
repoPerms os.FileMode
|
2017-10-24 22:21:15 +00:00
|
|
|
mu sync.Mutex
|
2017-10-19 00:18:29 +00:00
|
|
|
}
|
|
|
|
|
2017-10-25 15:49:46 +00:00
|
|
|
func (f *Filesystem) EachObject(fn func(Object) error) error {
|
|
|
|
var eachErr error
|
2019-09-17 19:20:04 +00:00
|
|
|
tools.FastWalkDir(f.LFSObjectDir(), func(parentDir string, info os.FileInfo, err error) {
|
2017-10-25 15:49:46 +00:00
|
|
|
if err != nil {
|
|
|
|
eachErr = err
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if eachErr != nil || info.IsDir() {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if oidRE.MatchString(info.Name()) {
|
|
|
|
fn(Object{Oid: info.Name(), Size: info.Size()})
|
|
|
|
}
|
|
|
|
})
|
|
|
|
return eachErr
|
|
|
|
}
|
|
|
|
|
2017-10-25 01:20:09 +00:00
|
|
|
func (f *Filesystem) ObjectExists(oid string, size int64) bool {
|
2021-09-27 12:33:22 +00:00
|
|
|
if size == 0 {
|
|
|
|
return true
|
|
|
|
}
|
2017-10-25 17:31:15 +00:00
|
|
|
return tools.FileExistsOfSize(f.ObjectPathname(oid), size)
|
2017-10-25 01:20:09 +00:00
|
|
|
}
|
|
|
|
|
2017-10-25 17:31:15 +00:00
|
|
|
func (f *Filesystem) ObjectPath(oid string) (string, error) {
|
2020-11-16 15:30:18 +00:00
|
|
|
if len(oid) < 4 {
|
2021-12-14 16:05:03 +00:00
|
|
|
return "", errors.New(tr.Tr.Get("too short object ID: %q", oid))
|
2020-11-16 15:30:18 +00:00
|
|
|
}
|
2021-09-27 12:33:22 +00:00
|
|
|
if oid == EmptyObjectSHA256 {
|
|
|
|
return os.DevNull, nil
|
|
|
|
}
|
2017-10-25 17:31:15 +00:00
|
|
|
dir := f.localObjectDir(oid)
|
2018-12-05 16:15:52 +00:00
|
|
|
if err := tools.MkdirAll(dir, f); err != nil {
|
2021-12-14 16:05:03 +00:00
|
|
|
return "", errors.New(tr.Tr.Get("error trying to create local storage directory in %q: %s", dir, err))
|
2017-10-25 17:31:15 +00:00
|
|
|
}
|
|
|
|
return filepath.Join(dir, oid), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *Filesystem) ObjectPathname(oid string) string {
|
2021-09-27 12:33:22 +00:00
|
|
|
if oid == EmptyObjectSHA256 {
|
|
|
|
return os.DevNull
|
|
|
|
}
|
2017-10-25 01:16:14 +00:00
|
|
|
return filepath.Join(f.localObjectDir(oid), oid)
|
|
|
|
}
|
|
|
|
|
2018-06-13 14:56:46 +00:00
|
|
|
func (f *Filesystem) DecodePathname(path string) string {
|
|
|
|
return string(DecodePathBytes([]byte(path)))
|
2018-06-25 04:49:53 +00:00
|
|
|
}
|
|
|
|
|
2018-12-04 22:25:02 +00:00
|
|
|
func (f *Filesystem) RepositoryPermissions(executable bool) os.FileMode {
|
|
|
|
if executable {
|
|
|
|
return tools.ExecutablePermissions(f.repoPerms)
|
|
|
|
}
|
|
|
|
return f.repoPerms
|
|
|
|
}
|
|
|
|
|
2018-06-25 04:49:53 +00:00
|
|
|
/**
|
2022-01-05 06:49:08 +00:00
|
|
|
* Revert non ascii characters escaped by git or windows (as octal sequences \000) back to bytes.
|
2018-06-25 04:49:53 +00:00
|
|
|
*/
|
2018-06-13 14:56:46 +00:00
|
|
|
func DecodePathBytes(path []byte) []byte {
|
2018-06-25 04:49:53 +00:00
|
|
|
var expression = regexp.MustCompile(`\\[0-9]{3}`)
|
|
|
|
var buffer bytes.Buffer
|
2018-06-13 14:56:46 +00:00
|
|
|
|
2018-06-25 04:49:53 +00:00
|
|
|
// strip quotes if any
|
2018-06-13 14:56:46 +00:00
|
|
|
if len(path) > 2 && path[0] == '"' && path[len(path)-1] == '"' {
|
|
|
|
path = path[1 : len(path)-1]
|
2018-06-25 04:49:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
base := 0
|
2018-06-13 14:56:46 +00:00
|
|
|
for _, submatches := range expression.FindAllSubmatchIndex(path, -1) {
|
|
|
|
buffer.Write(path[base:submatches[0]])
|
|
|
|
|
|
|
|
match := string(path[submatches[0]+1 : submatches[0]+4])
|
2018-06-25 04:49:53 +00:00
|
|
|
|
|
|
|
k, err := strconv.ParseUint(match, 8, 64)
|
2018-06-13 14:56:46 +00:00
|
|
|
if err != nil {
|
|
|
|
return path
|
|
|
|
} // abort on error
|
2018-06-25 04:49:53 +00:00
|
|
|
|
2018-06-13 14:56:46 +00:00
|
|
|
buffer.Write([]byte{byte(k)})
|
2018-06-25 04:49:53 +00:00
|
|
|
base = submatches[1]
|
2018-06-13 14:56:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
buffer.Write(path[base:len(path)])
|
2018-06-25 04:49:53 +00:00
|
|
|
|
|
|
|
return buffer.Bytes()
|
|
|
|
}
|
|
|
|
|
2017-10-25 01:16:14 +00:00
|
|
|
func (f *Filesystem) localObjectDir(oid string) string {
|
|
|
|
return filepath.Join(f.LFSObjectDir(), oid[0:2], oid[2:4])
|
|
|
|
}
|
|
|
|
|
2018-07-09 15:28:23 +00:00
|
|
|
func (f *Filesystem) ObjectReferencePaths(oid string) []string {
|
|
|
|
if len(f.ReferenceDirs) == 0 {
|
|
|
|
return nil
|
2017-10-24 21:35:14 +00:00
|
|
|
}
|
|
|
|
|
2018-07-09 15:28:23 +00:00
|
|
|
var paths []string
|
|
|
|
for _, ref := range f.ReferenceDirs {
|
|
|
|
paths = append(paths, filepath.Join(ref, oid[0:2], oid[2:4], oid))
|
|
|
|
}
|
|
|
|
return paths
|
2017-10-24 21:35:14 +00:00
|
|
|
}
|
|
|
|
|
2017-10-25 01:16:14 +00:00
|
|
|
func (f *Filesystem) LFSObjectDir() string {
|
|
|
|
f.mu.Lock()
|
|
|
|
defer f.mu.Unlock()
|
|
|
|
|
|
|
|
if len(f.lfsobjdir) == 0 {
|
|
|
|
f.lfsobjdir = filepath.Join(f.LFSStorageDir, "objects")
|
2018-12-05 16:15:52 +00:00
|
|
|
tools.MkdirAll(f.lfsobjdir, f)
|
2017-10-25 01:16:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return f.lfsobjdir
|
|
|
|
}
|
|
|
|
|
2017-10-24 22:21:15 +00:00
|
|
|
func (f *Filesystem) LogDir() string {
|
|
|
|
f.mu.Lock()
|
|
|
|
defer f.mu.Unlock()
|
|
|
|
|
|
|
|
if len(f.logdir) == 0 {
|
|
|
|
f.logdir = filepath.Join(f.LFSStorageDir, "logs")
|
2018-12-05 16:15:52 +00:00
|
|
|
tools.MkdirAll(f.logdir, f)
|
2017-10-24 22:21:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return f.logdir
|
|
|
|
}
|
|
|
|
|
2017-10-25 00:59:36 +00:00
|
|
|
func (f *Filesystem) TempDir() string {
|
|
|
|
f.mu.Lock()
|
|
|
|
defer f.mu.Unlock()
|
|
|
|
|
|
|
|
if len(f.tmpdir) == 0 {
|
|
|
|
f.tmpdir = filepath.Join(f.LFSStorageDir, "tmp")
|
2018-12-05 16:15:52 +00:00
|
|
|
tools.MkdirAll(f.tmpdir, f)
|
2017-10-25 00:59:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return f.tmpdir
|
|
|
|
}
|
|
|
|
|
2017-10-25 01:16:14 +00:00
|
|
|
func (f *Filesystem) Cleanup() error {
|
|
|
|
if f == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return f.cleanupTmp()
|
|
|
|
}
|
|
|
|
|
2017-10-24 21:22:13 +00:00
|
|
|
// New initializes a new *Filesystem with the given directories. gitdir is the
|
|
|
|
// path to the bare repo, workdir is the path to the repository working
|
|
|
|
// directory, and lfsdir is the optional path to the `.git/lfs` directory.
|
2018-12-04 22:25:02 +00:00
|
|
|
// repoPerms is the permissions for directories in the repository.
|
|
|
|
func New(env Environment, gitdir, workdir, lfsdir string, repoPerms os.FileMode) *Filesystem {
|
2017-10-24 21:22:13 +00:00
|
|
|
fs := &Filesystem{
|
2017-10-24 21:58:42 +00:00
|
|
|
GitStorageDir: resolveGitStorageDir(gitdir),
|
2017-10-19 00:35:03 +00:00
|
|
|
}
|
2017-10-24 21:22:13 +00:00
|
|
|
|
2018-07-09 16:23:27 +00:00
|
|
|
fs.ReferenceDirs = resolveReferenceDirs(env, fs.GitStorageDir)
|
2017-10-24 21:22:13 +00:00
|
|
|
|
2017-10-24 21:58:42 +00:00
|
|
|
if len(lfsdir) == 0 {
|
|
|
|
lfsdir = "lfs"
|
|
|
|
}
|
|
|
|
|
|
|
|
if filepath.IsAbs(lfsdir) {
|
|
|
|
fs.LFSStorageDir = lfsdir
|
|
|
|
} else {
|
|
|
|
fs.LFSStorageDir = filepath.Join(fs.GitStorageDir, lfsdir)
|
|
|
|
}
|
|
|
|
|
2018-12-04 22:25:02 +00:00
|
|
|
fs.repoPerms = repoPerms
|
|
|
|
|
2017-10-24 21:22:13 +00:00
|
|
|
return fs
|
2016-05-13 16:38:06 +00:00
|
|
|
}
|
|
|
|
|
2018-07-09 16:23:27 +00:00
|
|
|
func resolveReferenceDirs(env Environment, gitStorageDir string) []string {
|
2018-07-09 15:28:23 +00:00
|
|
|
var references []string
|
|
|
|
|
2018-07-09 16:23:27 +00:00
|
|
|
envAlternates, ok := env.Get("GIT_ALTERNATE_OBJECT_DIRECTORIES")
|
|
|
|
if ok {
|
|
|
|
splits := strings.Split(envAlternates, string(os.PathListSeparator))
|
|
|
|
for _, split := range splits {
|
|
|
|
if dir, ok := existsAlternate(split); ok {
|
|
|
|
references = append(references, dir)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
cloneReferencePath := filepath.Join(gitStorageDir, "objects", "info", "alternates")
|
|
|
|
if tools.FileExists(cloneReferencePath) {
|
2018-07-09 15:28:23 +00:00
|
|
|
f, err := os.Open(cloneReferencePath)
|
|
|
|
if err != nil {
|
|
|
|
tracerx.Printf("could not open %s: %s",
|
|
|
|
cloneReferencePath, err)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
|
|
for scanner.Scan() {
|
|
|
|
text := strings.TrimSpace(scanner.Text())
|
2018-07-16 15:56:40 +00:00
|
|
|
if len(text) == 0 || strings.HasPrefix(text, "#") {
|
2018-07-09 15:28:23 +00:00
|
|
|
continue
|
2016-05-13 16:38:06 +00:00
|
|
|
}
|
2018-07-09 15:28:23 +00:00
|
|
|
|
2018-07-09 16:22:19 +00:00
|
|
|
if dir, ok := existsAlternate(text); ok {
|
|
|
|
references = append(references, dir)
|
2018-07-09 15:28:23 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := scanner.Err(); err != nil {
|
|
|
|
tracerx.Printf("could not scan %s: %s",
|
|
|
|
cloneReferencePath, err)
|
2016-05-13 16:38:06 +00:00
|
|
|
}
|
|
|
|
}
|
2018-07-09 15:28:23 +00:00
|
|
|
return references
|
2016-05-13 16:38:06 +00:00
|
|
|
}
|
|
|
|
|
2018-07-09 16:22:19 +00:00
|
|
|
// existsAlternate takes an object directory given in "objs" (read as a single,
|
|
|
|
// line from .git/objects/info/alternates). If that is a satisfiable alternates
|
|
|
|
// directory (i.e., it exists), the directory is returned along with "true". If
|
|
|
|
// not, the empty string and false is returned instead.
|
|
|
|
func existsAlternate(objs string) (string, bool) {
|
|
|
|
objs = strings.TrimSpace(objs)
|
2018-07-16 18:32:54 +00:00
|
|
|
if strings.HasPrefix(objs, "\"") {
|
2018-07-09 16:22:19 +00:00
|
|
|
var err error
|
|
|
|
|
2018-07-16 15:56:40 +00:00
|
|
|
unquote := strings.LastIndex(objs, "\"")
|
2018-07-09 16:22:19 +00:00
|
|
|
if unquote == 0 {
|
2018-07-16 15:56:40 +00:00
|
|
|
return "", false
|
2018-07-09 16:22:19 +00:00
|
|
|
}
|
|
|
|
|
2018-07-16 15:56:40 +00:00
|
|
|
objs, err = strconv.Unquote(objs[:unquote+1])
|
2018-07-09 16:22:19 +00:00
|
|
|
if err != nil {
|
|
|
|
return "", false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
storage := filepath.Join(filepath.Dir(objs), "lfs", "objects")
|
|
|
|
|
|
|
|
if tools.DirExists(storage) {
|
|
|
|
return storage, true
|
|
|
|
}
|
|
|
|
return "", false
|
|
|
|
}
|
|
|
|
|
2016-05-13 16:38:06 +00:00
|
|
|
// From a git dir, get the location that objects are to be stored (we will store lfs alongside)
|
|
|
|
// Sometimes there is an additional level of redirect on the .git folder by way of a commondir file
|
|
|
|
// before you find object storage, e.g. 'git worktree' uses this. It redirects to gitdir either by GIT_DIR
|
|
|
|
// (during setup) or .git/git-dir: (during use), but this only contains the index etc, the objects
|
|
|
|
// are found in another git dir via 'commondir'.
|
|
|
|
func resolveGitStorageDir(gitDir string) string {
|
|
|
|
commondirpath := filepath.Join(gitDir, "commondir")
|
|
|
|
if tools.FileExists(commondirpath) && !tools.DirExists(filepath.Join(gitDir, "objects")) {
|
|
|
|
// no git-dir: prefix in commondir
|
|
|
|
storage, err := processGitRedirectFile(commondirpath, "")
|
|
|
|
if err == nil {
|
|
|
|
return storage
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return gitDir
|
|
|
|
}
|
|
|
|
|
|
|
|
func processGitRedirectFile(file, prefix string) (string, error) {
|
|
|
|
data, err := ioutil.ReadFile(file)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
contents := string(data)
|
|
|
|
var dir string
|
|
|
|
if len(prefix) > 0 {
|
|
|
|
if !strings.HasPrefix(contents, prefix) {
|
|
|
|
// Prefix required & not found
|
|
|
|
return "", nil
|
|
|
|
}
|
|
|
|
dir = strings.TrimSpace(contents[len(prefix):])
|
|
|
|
} else {
|
|
|
|
dir = strings.TrimSpace(contents)
|
|
|
|
}
|
|
|
|
|
|
|
|
if !filepath.IsAbs(dir) {
|
|
|
|
// The .git file contains a relative path.
|
|
|
|
// Create an absolute path based on the directory the .git file is located in.
|
|
|
|
dir = filepath.Join(filepath.Dir(file), dir)
|
|
|
|
}
|
|
|
|
|
|
|
|
return dir, nil
|
|
|
|
}
|