2016-12-14 22:17:24 +00:00
|
|
|
package commands
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
|
|
|
"io"
|
2016-12-14 22:29:03 +00:00
|
|
|
"os"
|
2016-12-14 22:17:24 +00:00
|
|
|
"sync"
|
|
|
|
|
2021-09-01 19:41:10 +00:00
|
|
|
"github.com/git-lfs/git-lfs/v3/config"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/errors"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/git"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/lfs"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/subprocess"
|
|
|
|
"github.com/git-lfs/git-lfs/v3/tq"
|
2022-01-03 20:46:40 +00:00
|
|
|
"github.com/git-lfs/git-lfs/v3/tr"
|
2016-12-14 22:17:24 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// Handles the process of checking out a single file, and updating the git
|
|
|
|
// index.
|
2017-09-27 23:11:42 +00:00
|
|
|
func newSingleCheckout(gitEnv config.Environment, remote string) abstractCheckout {
|
|
|
|
clean, ok := gitEnv.Get("filter.lfs.clean")
|
|
|
|
if !ok || len(clean) == 0 {
|
commands: avoid remote connections in checkout
Right now, creating a checkout object as used by the checkout and pull
commands creates a manifest. This manifest in turn creates an
SSHTransfer object to potentially connect via the new pure SSH protocol.
However, connecting to the Internet during `git lfs checkout` is
unexpected and may cause this operation to fail when one is not online.
In order to set the manifest's configuration parameters correctly, we
need to know whether it's possible to make a pure SSH protocol
connection, so it's non-trivial to avoid this connection by delaying the
initialization of SSHTransfer object. Instead, let's lazily create our
manifest in the checkout code to avoid creating the connection. If
we're performing a `git lfs pull`, then it will be used and we'll
initialize it very quickly, making the connection, and if we're using
`git lfs checkout`, it won't be initialized at all, and we'll remain
offline.
2022-12-16 16:28:09 +00:00
|
|
|
return &noOpCheckout{remote: remote}
|
2017-09-27 23:11:42 +00:00
|
|
|
}
|
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
// Get a converter from repo-relative to cwd-relative
|
|
|
|
// Since writing data & calling git update-index must be relative to cwd
|
2017-10-19 00:09:33 +00:00
|
|
|
pathConverter, err := lfs.NewRepoToCurrentPathConverter(cfg)
|
2016-12-14 22:17:24 +00:00
|
|
|
if err != nil {
|
2022-01-03 20:46:40 +00:00
|
|
|
Panic(err, tr.Tr.Get("Could not convert file paths"))
|
2016-12-14 22:17:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return &singleCheckout{
|
|
|
|
gitIndexer: &gitIndexer{},
|
|
|
|
pathConverter: pathConverter,
|
commands: avoid remote connections in checkout
Right now, creating a checkout object as used by the checkout and pull
commands creates a manifest. This manifest in turn creates an
SSHTransfer object to potentially connect via the new pure SSH protocol.
However, connecting to the Internet during `git lfs checkout` is
unexpected and may cause this operation to fail when one is not online.
In order to set the manifest's configuration parameters correctly, we
need to know whether it's possible to make a pure SSH protocol
connection, so it's non-trivial to avoid this connection by delaying the
initialization of SSHTransfer object. Instead, let's lazily create our
manifest in the checkout code to avoid creating the connection. If
we're performing a `git lfs pull`, then it will be used and we'll
initialize it very quickly, making the connection, and if we're using
`git lfs checkout`, it won't be initialized at all, and we'll remain
offline.
2022-12-16 16:28:09 +00:00
|
|
|
manifest: nil,
|
|
|
|
remote: remote,
|
2016-12-14 22:17:24 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-09-27 23:11:42 +00:00
|
|
|
type abstractCheckout interface {
|
|
|
|
Manifest() *tq.Manifest
|
|
|
|
Skip() bool
|
|
|
|
Run(*lfs.WrappedPointer)
|
2018-10-01 18:37:45 +00:00
|
|
|
RunToPath(*lfs.WrappedPointer, string) error
|
2017-09-27 23:11:42 +00:00
|
|
|
Close()
|
|
|
|
}
|
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
type singleCheckout struct {
|
|
|
|
gitIndexer *gitIndexer
|
|
|
|
pathConverter lfs.PathConverter
|
2016-12-16 17:35:38 +00:00
|
|
|
manifest *tq.Manifest
|
commands: avoid remote connections in checkout
Right now, creating a checkout object as used by the checkout and pull
commands creates a manifest. This manifest in turn creates an
SSHTransfer object to potentially connect via the new pure SSH protocol.
However, connecting to the Internet during `git lfs checkout` is
unexpected and may cause this operation to fail when one is not online.
In order to set the manifest's configuration parameters correctly, we
need to know whether it's possible to make a pure SSH protocol
connection, so it's non-trivial to avoid this connection by delaying the
initialization of SSHTransfer object. Instead, let's lazily create our
manifest in the checkout code to avoid creating the connection. If
we're performing a `git lfs pull`, then it will be used and we'll
initialize it very quickly, making the connection, and if we're using
`git lfs checkout`, it won't be initialized at all, and we'll remain
offline.
2022-12-16 16:28:09 +00:00
|
|
|
remote string
|
2016-12-14 22:17:24 +00:00
|
|
|
}
|
|
|
|
|
2017-09-27 23:11:42 +00:00
|
|
|
func (c *singleCheckout) Manifest() *tq.Manifest {
|
commands: avoid remote connections in checkout
Right now, creating a checkout object as used by the checkout and pull
commands creates a manifest. This manifest in turn creates an
SSHTransfer object to potentially connect via the new pure SSH protocol.
However, connecting to the Internet during `git lfs checkout` is
unexpected and may cause this operation to fail when one is not online.
In order to set the manifest's configuration parameters correctly, we
need to know whether it's possible to make a pure SSH protocol
connection, so it's non-trivial to avoid this connection by delaying the
initialization of SSHTransfer object. Instead, let's lazily create our
manifest in the checkout code to avoid creating the connection. If
we're performing a `git lfs pull`, then it will be used and we'll
initialize it very quickly, making the connection, and if we're using
`git lfs checkout`, it won't be initialized at all, and we'll remain
offline.
2022-12-16 16:28:09 +00:00
|
|
|
if c.manifest == nil {
|
|
|
|
c.manifest = getTransferManifestOperationRemote("download", c.remote)
|
|
|
|
}
|
2017-09-27 23:11:42 +00:00
|
|
|
return c.manifest
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *singleCheckout) Skip() bool {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
func (c *singleCheckout) Run(p *lfs.WrappedPointer) {
|
2017-10-03 16:48:18 +00:00
|
|
|
cwdfilepath := c.pathConverter.Convert(p.Name)
|
|
|
|
|
2016-12-14 22:29:03 +00:00
|
|
|
// Check the content - either missing or still this pointer (not exist is ok)
|
2017-10-03 16:48:18 +00:00
|
|
|
filepointer, err := lfs.DecodePointerFromFile(cwdfilepath)
|
2016-12-14 22:29:03 +00:00
|
|
|
if err != nil && !os.IsNotExist(err) {
|
2020-10-22 20:56:31 +00:00
|
|
|
if errors.IsNotAPointerError(err) || errors.IsBadPointerKeyError(err) {
|
2016-12-14 22:29:03 +00:00
|
|
|
// File has non-pointer content, leave it alone
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2022-01-03 20:46:40 +00:00
|
|
|
LoggedError(err, tr.Tr.Get("Checkout error: %s", err))
|
2016-12-14 22:29:03 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if filepointer != nil && filepointer.Oid != p.Oid {
|
|
|
|
// User has probably manually reset a file to another commit
|
|
|
|
// while leaving it a pointer; don't mess with this
|
|
|
|
return
|
2016-12-14 22:17:24 +00:00
|
|
|
}
|
|
|
|
|
2018-10-01 18:37:45 +00:00
|
|
|
if err := c.RunToPath(p, cwdfilepath); err != nil {
|
2016-12-14 22:29:03 +00:00
|
|
|
if errors.IsDownloadDeclinedError(err) {
|
|
|
|
// acceptable error, data not local (fetch not run or include/exclude)
|
2022-01-03 20:46:40 +00:00
|
|
|
Error(tr.Tr.Get("Skipped checkout for %q, content not local. Use fetch to download.", p.Name))
|
2016-12-14 22:29:03 +00:00
|
|
|
} else {
|
2022-01-03 20:46:40 +00:00
|
|
|
FullError(errors.New(tr.Tr.Get("could not check out %q", p.Name)))
|
2016-12-14 22:17:24 +00:00
|
|
|
}
|
2016-12-14 22:29:03 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// errors are only returned when the gitIndexer is starting a new cmd
|
|
|
|
if err := c.gitIndexer.Add(cwdfilepath); err != nil {
|
2022-01-03 20:46:40 +00:00
|
|
|
Panic(err, tr.Tr.Get("Could not update the index"))
|
2016-12-14 22:17:24 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-01 18:37:45 +00:00
|
|
|
// RunToPath checks out the pointer specified by p to the given path. It does
|
|
|
|
// not perform any sort of sanity checking or add the path to the index.
|
|
|
|
func (c *singleCheckout) RunToPath(p *lfs.WrappedPointer, path string) error {
|
|
|
|
gitfilter := lfs.NewGitFilter(cfg)
|
|
|
|
return gitfilter.SmudgeToFile(path, p.Pointer, false, c.manifest, nil)
|
|
|
|
}
|
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
func (c *singleCheckout) Close() {
|
|
|
|
if err := c.gitIndexer.Close(); err != nil {
|
commands,lfs*: remove most newlines from messages
In general, we can make the translation task somewhat
easier for translators, and less prone to error, if we
remove non-language-specific newlines from the text strings
to be translated.
To do this we can in some cases simply drop newlines (e.g.,
in panic messages), or break multi-line text strings into
several separate messages where that makes sense. For the
most part we simply append the necessary trailing or
intermediate newlines using the formatting tools available,
especially those provided in the commands/commands.go file.
Note that those tools (i.e., Print(), Exit(), etc.) always
append a newline, but some messages are intended to have
two final newlines, such as those from the "git lfs status"
command.
We also add a note to translators regarding some specific
spacing required by a message output by the "git lfs dedup"
command.
2022-01-27 06:39:31 +00:00
|
|
|
LoggedError(err, "%s\n%s", tr.Tr.Get("Error updating the Git index:"), c.gitIndexer.Output())
|
2016-12-14 22:17:24 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-09-27 23:11:42 +00:00
|
|
|
type noOpCheckout struct {
|
|
|
|
manifest *tq.Manifest
|
commands: avoid remote connections in checkout
Right now, creating a checkout object as used by the checkout and pull
commands creates a manifest. This manifest in turn creates an
SSHTransfer object to potentially connect via the new pure SSH protocol.
However, connecting to the Internet during `git lfs checkout` is
unexpected and may cause this operation to fail when one is not online.
In order to set the manifest's configuration parameters correctly, we
need to know whether it's possible to make a pure SSH protocol
connection, so it's non-trivial to avoid this connection by delaying the
initialization of SSHTransfer object. Instead, let's lazily create our
manifest in the checkout code to avoid creating the connection. If
we're performing a `git lfs pull`, then it will be used and we'll
initialize it very quickly, making the connection, and if we're using
`git lfs checkout`, it won't be initialized at all, and we'll remain
offline.
2022-12-16 16:28:09 +00:00
|
|
|
remote string
|
2017-09-27 23:11:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *noOpCheckout) Manifest() *tq.Manifest {
|
commands: avoid remote connections in checkout
Right now, creating a checkout object as used by the checkout and pull
commands creates a manifest. This manifest in turn creates an
SSHTransfer object to potentially connect via the new pure SSH protocol.
However, connecting to the Internet during `git lfs checkout` is
unexpected and may cause this operation to fail when one is not online.
In order to set the manifest's configuration parameters correctly, we
need to know whether it's possible to make a pure SSH protocol
connection, so it's non-trivial to avoid this connection by delaying the
initialization of SSHTransfer object. Instead, let's lazily create our
manifest in the checkout code to avoid creating the connection. If
we're performing a `git lfs pull`, then it will be used and we'll
initialize it very quickly, making the connection, and if we're using
`git lfs checkout`, it won't be initialized at all, and we'll remain
offline.
2022-12-16 16:28:09 +00:00
|
|
|
if c.manifest == nil {
|
|
|
|
c.manifest = getTransferManifestOperationRemote("download", c.remote)
|
|
|
|
}
|
2017-09-27 23:11:42 +00:00
|
|
|
return c.manifest
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *noOpCheckout) Skip() bool {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2018-10-01 18:37:45 +00:00
|
|
|
func (c *noOpCheckout) RunToPath(p *lfs.WrappedPointer, path string) error {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-09-27 23:11:42 +00:00
|
|
|
func (c *noOpCheckout) Run(p *lfs.WrappedPointer) {}
|
|
|
|
func (c *noOpCheckout) Close() {}
|
|
|
|
|
2016-12-14 22:17:24 +00:00
|
|
|
// Don't fire up the update-index command until we have at least one file to
|
|
|
|
// give it. Otherwise git interprets the lack of arguments to mean param-less update-index
|
|
|
|
// which can trigger entire working copy to be re-examined, which triggers clean filters
|
|
|
|
// and which has unexpected side effects (e.g. downloading filtered-out files)
|
|
|
|
type gitIndexer struct {
|
2017-08-02 18:48:48 +00:00
|
|
|
cmd *subprocess.Cmd
|
2016-12-14 22:17:24 +00:00
|
|
|
input io.WriteCloser
|
|
|
|
output bytes.Buffer
|
|
|
|
mu sync.Mutex
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *gitIndexer) Add(path string) error {
|
|
|
|
i.mu.Lock()
|
|
|
|
defer i.mu.Unlock()
|
|
|
|
|
|
|
|
if i.cmd == nil {
|
|
|
|
// Fire up the update-index command
|
subprocess: report errors when finding executables
On Windows, when spawning a process, Go first looks for an
executable file with the correct name in the current directory, and
only if it fails to find one there does it look in the directories
listed in the PATH environment variable. We would prefer not to
replicate this behaviour and instead search only in the directories
in PATH. Therefore, starting with the mitigation of CVE-2020-27955
in commit 74d5f2397f9abe4834bf1fe1fa02fd6c141b77ce, we resolve paths
to executables ourselves rather than rely on Go to do so.
The subprocess.LookPath() function we introduced in that change
is adapted from Go's os/exec package. When it cannot find an
executable in PATH, it returns an empty path string and an
exec.ErrNotFound error. When that happens, we do not detect the
condition and simply set the command path to the empty string.
This can lead to undesirable behaviour in which a bug in the Go
os/exec library causes it to run another executable other than
the one we intended.
When we set the command path to the empty string and then ask to
execute the command, the native Go version of the LookPath() function
for Windows is run with an empty path because filepath.Base() returns
"." when passed the empty string and because we have left the current
working directory also set to an empty string:
https://github.com/golang/go/blob/1724077b789ad92972ab1ac03788389645306cbb/src/os/exec/exec.go#L348-L353
Since the path string does not contain any path separator
characters the current working directory is searched to try to
find a file with a matching name and executable file extension
(e.g., ".exe" or ".com"). To search the current working directory,
the "." directory name is prepended to the given path with
filepath.Join():
https://github.com/golang/go/blob/1724077b789ad92972ab1ac03788389645306cbb/src/os/exec/lp_windows.go#L84
The filepath.Join() function ignores empty arguments, so this
results in an incorrect filename of ".". All potential executable
file extensions from the PATHEXT environment variable (or from a
fixed list if that variable is not defined) are then appended to
this filename, including their leading dot separator characters,
thus producing filenames like "..com" and "..exe":
https://github.com/golang/go/blob/1724077b789ad92972ab1ac03788389645306cbb/src/os/exec/lp_windows.go#L46-L50
Should a file with one of these names exist in the current working
directory, its name will be returned, which means that it will be
executed instead of the command we expected to run. This is
obviously undesirable and presents a risk to users.
(Note that batch files named "..bat" and "..cmd" will also be found
in the same manner, but they will not actually be executed due to an
undocumented feature of the Windows API's family of CreateProcess*()
functions, which are used by Go to spawn processes. When passed an
lpApplicationName argument ending with a ".bat" or ".cmd" extension,
the CreateProcess*() functions appear to instead execute "cmd.exe"
and construct a "/c" command string from the lpCommandLine argument.
The value of that argument is set using the full command line we are
attempting to execute, and as such its first element is the actual
name of the executable we intended to run; therefore, the command
interpreter attempts to run the executable as a batch script and
fails, and the "..bat" or "..cmd" file is effectively ignored.)
To recap, when Git LFS attempts to execute a program on Windows,
if the executable is not found anywhere in PATH but does exist in
the current working directory, then when we call exec.Command()
Go's internal LookPath() function will find the executable and not
set the internal lookPathErr flag:
https://github.com/golang/go/blob/1724077b789ad92972ab1ac03788389645306cbb/src/os/exec/exec.go#L174-L179
Since we do not want to run executables in the current working
directory, we subsequently run our own LookPath() function, which
we use to reset the cmd.Path field. However, when our LookPath()
returns an empty string, we do not detect that and instead set
cmd.Path to that value. Then when we ask Go to run the command,
because lookPathErr is nil, it proceeds, and the empty path causes
it to find and run the first matching file in the working directory
named "..com" or "..exe" (or any similar name with an executable
file extension except for "..bat" and "..cmd").
We can prevent this behaviour by detecting when our LookPath()
function returns an error and propagating it upwards to all
callers of our subprocess.ExecCommand() function. We also add
checks for this error at appropriate points and log or report
the error as necessary.
One particular circumstance of note occurs when a user has a
Cygwin-installed "uname" in their PATH but not "cygpath",
but "cygpath.exe" does exist in their current working directory.
Then we will attempt to execute "cygpath" because we use the
presence of "uname" as an indication that Cygwin is fully installed.
Should a "..exe" or similar file also be present in the working
directory, then it will be executed instead of "cygpath.exe".
As with all other instances where we call subprocess.ExecCommand(),
in tools.translateCygwinPath() we will now check for a returned
error before trying to actually execute "cygpath". Unlike many
of the other cases, though, we do not need to report the error in
this one; instead, we simply return the current path from
translateCygwinPath() instead of canonicalizing it, just as we do
already if the "cygpath" executable fails for some reason.
Finally, we add a new test to t/t-path.sh which checks for the
incorrect execution of a "..exe" binary on Windows when "git.exe"
is not found in PATH but does exist in the current working
directory. This test passes when run with a Git LFS binary that
includes the remediations from this commit, and fails otherwise.
For our "malicious" binary named "..exe" we make use of the
lfstest-badpathcheck test helper we added in a previous commit.
We only run this test on Windows because the underlying bug in
Go is Windows-specific as it depends on path extensions from
PATHEXT being appended to the file name ".".
Co-authored-by: brian m. carlson <bk2204@github.com>
2022-03-29 06:16:16 +00:00
|
|
|
cmd, err := git.UpdateIndexFromStdin()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-08-26 10:39:42 +00:00
|
|
|
cmd.Stdout = &i.output
|
|
|
|
cmd.Stderr = &i.output
|
|
|
|
stdin, err := cmd.StdinPipe()
|
2016-12-14 22:17:24 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-08-26 10:39:42 +00:00
|
|
|
err = cmd.Start()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
i.cmd = cmd
|
2016-12-14 22:17:24 +00:00
|
|
|
i.input = stdin
|
|
|
|
}
|
|
|
|
|
|
|
|
i.input.Write([]byte(path + "\n"))
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *gitIndexer) Output() string {
|
|
|
|
return i.output.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *gitIndexer) Close() error {
|
|
|
|
i.mu.Lock()
|
|
|
|
defer i.mu.Unlock()
|
|
|
|
|
|
|
|
if i.input != nil {
|
|
|
|
i.input.Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
if i.cmd != nil {
|
|
|
|
return i.cmd.Wait()
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|