diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dd24e8bc..166d0911 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,6 @@ In general, contributors should develop on branches based off of `master` and pu 0. Create a new branch based on `master`: `git checkout -b master` 0. Make your change, add tests, and make sure the tests still pass 0. Push to your fork and [submit a pull request][pr] from your branch to `master` -0. Accept the [GitHub CLA][cla] 0. Pat yourself on the back and wait for your pull request to be reviewed Here are a few things you can do that will increase the likelihood of your pull request being accepted: @@ -161,4 +160,3 @@ v1.5 just shipped, set the version in master to `1.6-pre`, for example. [fork]: https://github.com/git-lfs/git-lfs/fork [pr]: https://github.com/git-lfs/git-lfs/compare [style]: https://github.com/golang/go/wiki/CodeReviewComments -[cla]: https://cla.github.com/git-lfs/git-lfs/accept diff --git a/README.md b/README.md index 1932fff1..07716375 100644 --- a/README.md +++ b/README.md @@ -11,86 +11,100 @@ [5]: https://ci.appveyor.com/api/projects/status/46a5yoqc3hk59bl5/branch/master?svg=true [6]: https://ci.appveyor.com/project/git-lfs/git-lfs/branch/master -Git LFS is a command line extension and [specification](docs/spec.md) for -managing large files with Git. The client is written in Go, with pre-compiled -binaries available for Mac, Windows, Linux, and FreeBSD. Check out the -[Git LFS website][page] for an overview of features. +[Git LFS](https://git-lfs.github.com) is a command line extension and +[specification](docs/spec.md) for managing large files with Git. -[page]: https://git-lfs.github.com/ +The client is written in Go, with pre-compiled binaries available for Mac, +Windows, Linux, and FreeBSD. Check out the [website](http://git-lfs.github.com) +for an overview of features. ## Getting Started -By default, the Git LFS client needs a Git LFS server to sync the large files -it manages. This works out of the box when using popular git repository -hosting providers like GitHub, Atlassian, etc. When you host your own -vanilla git server, for example, you need to either use a separate -[Git LFS server instance](https://github.com/git-lfs/git-lfs/wiki/Implementations), -or use the [custom transfer adapter](docs/custom-transfers.md) with -a transfer agent in blind mode, without having to use a Git LFS server instance. +### Installation -You can install the Git LFS client in several different ways, depending on -your setup and preferences. +You can install the Git LFS client in several different ways, depending on your +setup and preferences. -* Linux users can install Debian or RPM packages from [PackageCloud](https://packagecloud.io/github/git-lfs/install). See the [Installation Guide](./INSTALLING.md) for details. -* Mac users can install from [Homebrew](https://github.com/Homebrew/homebrew) with `brew install git-lfs`, or from [MacPorts](https://www.macports.org) with `port install git-lfs`. -* Windows users can install from [Chocolatey](https://chocolatey.org/) with `choco install git-lfs`. -* [Binary packages are available][rel] for Windows, Mac, Linux, and FreeBSD. -* You can build it with Go 1.8.1+. See the [Contributing Guide](./CONTRIBUTING.md) for instructions. +* **Linux users**. Debian and RPM packages are available from + [PackageCloud](https://packagecloud.io/github/git-lfs/install). +* **macOS users**. [Homebrew](https://brew.sh) bottles are distributed, and can + be installed via `brew install git-lfs`. +* **Windows users**. Chocolatey packages are distributed, and can be installed + via `choco install git-lfs`. -[rel]: https://github.com/git-lfs/git-lfs/releases +In addition, [binary packages](https://github.com/git-lfs/git-lfs/releases) are +available for Linux, macOS, Windows, and FreeBSD. This repository can also be +built from source using the latest version of [Go](https://golang.org), and the +available instructions in our +[Wiki](https://github.com/git-lfs/git-lfs/wiki/Installation#source). -Note: Git LFS requires Git v1.8.5 or higher. +### Usage -Once installed, you need to setup the global Git hooks for Git LFS. This only -needs to be done once per machine. +Git LFS requires a global installation once per-machine. This can be done by +running: ```bash $ git lfs install ``` -Now, it's time to add some large files to a repository. The first step is to -specify file patterns to store with Git LFS. These file patterns are stored in -`.gitattributes`. +To begin using Git LFS within your Git repository, you can indicate which files +you would like Git LFS to manage. This can be done by running the following +_from within Git repository_: ```bash -$ mkdir large-repo -$ cd large-repo -$ git init - -# Add all zip files through Git LFS -$ git lfs track "*.zip" +$ git lfs track "*.psd" ``` -Now you're ready to push some commits: +(Where `*.psd` is the pattern of filenames that you wish to track. You can read +more about this pattern syntax +[here](https://git-scm.com/docs/gitattributes)). + +After any invocation of `git-lfs-track(1)` or `git-lfs-untrack(1)`, you _must +commit changes to your `.gitattributes` file_. This can be done by running: ```bash $ git add .gitattributes -$ git add my.zip -$ git commit -m "add zip" +$ git commit -m "track *.psd files using Git LFS" ``` -You can confirm that Git LFS is managing your zip file: +You can now interact with your Git repository as usual, and Git LFS will take +care of managing your large files. For example, changing a file named `my.psd` +(tracked above via `*.psd`): + +```bash +$ git add my.psd +$ git commit -m "add psd" +``` + +> _Tip:_ if you have large files already in your repository's history, `git lfs +> track` will _not_ track them retroactively. To migrate existing large files +> in your history to use Git LFS, use `git lfs migrate`. For example: +> +> ``` +> $ git lfs migrate import --include="*.psd" +> ``` +> +> For more information, read [`git-lfs-migrate(1)`](https://github.com/git-lfs/git-lfs/blob/master/docs/man/git-lfs-migrate.1.ronn). + +You can confirm that Git LFS is managing your PSD file: ```bash $ git lfs ls-files -my.zip +3c2f7aedfb * my.psd ``` Once you've made your commits, push your files to the Git remote: ```bash $ git push origin master -Sending my.zip -LFS: 12.58 MB / 12.58 MB 100.00 % -Counting objects: 2, done. -Delta compression using up to 8 threads. -Compressing objects: 100% (5/5), done. -Writing objects: 100% (5/5), 548 bytes | 0 bytes/s, done. -Total 5 (delta 1), reused 0 (delta 0) +Uploading LFS objects: 100% (1/1), 810 B, 1.2 KB/s +# ... To https://github.com/git-lfs/git-lfs-test 67fcf6a..47b2002 master -> master ``` +Note: Git LFS requires Git v1.8.5 or higher. + ## Limitations Git LFS maintains a list of currently known limitations, which you can find and @@ -105,34 +119,21 @@ $ git lfs help ``` The [official documentation](docs) has command references and specifications for -the tool. You can ask questions in the [Git LFS chat room][chat], or [file a new -issue][ish]. Be sure to include details about the problem so we can -troubleshoot it. +the tool. -1. Include the output of `git lfs env`, which shows how your Git environment -is setup. -2. Include `GIT_TRACE=1` in any bad Git commands to enable debug messages. -3. If the output includes a message like `Errors logged to /path/to/.git/lfs/objects/logs/*.log`, -throw the contents in the issue, or as a link to a Gist or paste site. +You can always [open an issue](https://github.com/git-lfs/git-lfs/issues), and +one of the Core Team members will respond to you. Please be sure to include: -[chat]: https://gitter.im/git-lfs/git-lfs -[ish]: https://github.com/git-lfs/git-lfs/issues +1. The output of `git lfs env`, which displays helpful information about your + Git repository useful in debugging. +2. Any failed commands re-run with `GIT_TRACE=1` in the environment, which + displays additional information pertaining to why a command crashed. ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) for info on working on Git LFS and sending patches. Related projects are listed on the [Implementations wiki -page][impl]. You can also join [the project's chat room][chat]. - -[impl]: https://github.com/git-lfs/git-lfs/wiki/Implementations - -### Using LFS from other Go code - -At the moment git-lfs is only focussed on the stability of its command line -interface, and the [server APIs](docs/api/README.md). The contents of the -source packages is subject to change. We therefore currently discourage other -Go code from depending on the git-lfs packages directly; an API to be used by -external Go code may be provided in future. +page](https://github.com/git-lfs/git-lfs/wiki/Implementations). ## Core Team @@ -140,6 +141,32 @@ These are the humans that form the Git LFS core team, which runs the project. In alphabetical order: -| [@andyneff](https://github.com/andyneff) | [@rubyist](https://github.com/rubyist) | [@sinbad](https://github.com/sinbad) | [@technoweenie](https://github.com/technoweenie) | [@ttaylorr](https://github.com/ttaylorr) | -|---|---|---|---|---| -| [![](https://avatars1.githubusercontent.com/u/7596961?v=3&s=100)](https://github.com/andyneff) | [![](https://avatars1.githubusercontent.com/u/143?v=3&s=100)](https://github.com/rubyist) | [![](https://avatars1.githubusercontent.com/u/142735?v=3&s=100)](https://github.com/sinbad) | [![](https://avatars3.githubusercontent.com/u/21?v=3&s=100)](https://github.com/technoweenie) | [![](https://avatars3.githubusercontent.com/u/443245?v=3&s=100)](https://github.com/ttaylorr) | +| [@larsxschneider][larsxschneider-user] | [@ttaylorr][ttaylorr-user] | +|---|---| +| [![][larsxschneider-img]][larsxschneider-user] | [![][ttaylorr-img]][ttaylorr-user] | + +[larsxschneider-img]: https://avatars1.githubusercontent.com/u/477434?s=100&v=4 +[ttaylorr-img]: https://avatars2.githubusercontent.com/u/443245?s=100&v=4 +[larsxschneider-user]: https://github.com/larsxschneider +[ttaylorr-user]: https://github.com/ttaylorr + +### Alumni + +These are the humans that have in the past formed the Git LFS core team, or +have otherwise contributed a significant amount to the project. Git LFS would +not be possible without them. + +In alphabetical order: + +| [@andyneff][andyneff-user] | [@rubyist][rubyist-user] | [@sinbad][sinbad-user] | [@technoweenie][technoweenie-user] | +|---|---|---|---| +| [![][andyneff-img]][andyneff-user] | [![][rubyist-img]][rubyist-user] | [![][sinbad-img]][sinbad-user] | [![][technoweenie-img]][technoweenie-user] | + +[andyneff-img]: https://avatars1.githubusercontent.com/u/7596961?v=3&s=100 +[rubyist-img]: https://avatars1.githubusercontent.com/u/143?v=3&s=100 +[sinbad-img]: https://avatars1.githubusercontent.com/u/142735?v=3&s=100 +[technoweenie-img]: https://avatars3.githubusercontent.com/u/21?v=3&s=100 +[andyneff-user]: https://github.com/andyneff +[sinbad-user]: https://github.com/sinbad +[rubyist-user]: https://github.com/rubyist +[technoweenie-user]: https://github.com/technoweenie diff --git a/commands/command_checkout.go b/commands/command_checkout.go index d26b0026..122f2460 100644 --- a/commands/command_checkout.go +++ b/commands/command_checkout.go @@ -3,6 +3,7 @@ package commands import ( "fmt" "os" + "strings" "github.com/git-lfs/git-lfs/filepathfilter" "github.com/git-lfs/git-lfs/git" @@ -14,6 +15,15 @@ import ( func checkoutCommand(cmd *cobra.Command, args []string) { requireInRepo() + + msg := []string{ + "WARNING: 'git lfs checkout' is deprecated and will be removed in v3.0.0.", + + "'git checkout' has been updated in upstream Git to have comparable speeds", + "to 'git lfs checkout'.", + } + fmt.Fprintln(os.Stderr, strings.Join(msg, "\n")) + ref, err := git.CurrentRef() if err != nil { Panic(err, "Could not checkout") @@ -29,6 +39,7 @@ func checkoutCommand(cmd *cobra.Command, args []string) { var pointers []*lfs.WrappedPointer logger := tasklog.NewLogger(os.Stdout) meter := tq.NewMeter() + meter.Direction = tq.Checkout meter.Logger = meter.LoggerFromEnv(cfg.Os) logger.Enqueue(meter) chgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { diff --git a/commands/command_lock.go b/commands/command_lock.go index eb06104b..ab88d7de 100644 --- a/commands/command_lock.go +++ b/commands/command_lock.go @@ -79,6 +79,11 @@ func lockPath(file string) (string, error) { if err != nil { return "", err } + wd, err = filepath.EvalSymlinks(wd) + if err != nil { + return "", errors.Wrapf(err, + "could not follow symlinks for %s", wd) + } abs := filepath.Join(wd, file) path := strings.TrimPrefix(abs, repo) diff --git a/commands/command_migrate.go b/commands/command_migrate.go index 8aef8ab9..fe35c2f8 100644 --- a/commands/command_migrate.go +++ b/commands/command_migrate.go @@ -32,6 +32,17 @@ var ( // migrateVerbose enables verbose logging migrateVerbose bool + + // objectMapFile is the path to the map of old sha1 to new sha1 + // commits + objectMapFilePath string + + // migrateNoRewrite is the flag indicating whether or not the + // command should rewrite git history + migrateNoRewrite bool + // migrateCommitMessage is the message to use with the commit generated + // by the migrate command + migrateCommitMessage string ) // migrate takes the given command and arguments, *odb.ObjectDatabase, as well @@ -83,8 +94,9 @@ func rewriteOptions(args []string, opts *githistory.RewriteOptions, l *tasklog.L Include: include, Exclude: exclude, - UpdateRefs: opts.UpdateRefs, - Verbose: opts.Verbose, + UpdateRefs: opts.UpdateRefs, + Verbose: opts.Verbose, + ObjectMapFilePath: opts.ObjectMapFilePath, BlobFn: opts.BlobFn, TreeCallbackFn: opts.TreeCallbackFn, @@ -281,6 +293,9 @@ func init() { importCmd := NewCommand("import", migrateImportCommand) importCmd.Flags().BoolVar(&migrateVerbose, "verbose", false, "Verbose logging") + importCmd.Flags().StringVar(&objectMapFilePath, "object-map", "", "Object map file") + importCmd.Flags().BoolVar(&migrateNoRewrite, "no-rewrite", false, "Add new history without rewriting previous") + importCmd.Flags().StringVarP(&migrateCommitMessage, "message", "m", "", "With --no-rewrite, an optional commit message") RegisterCommand("migrate", nil, func(cmd *cobra.Command) { cmd.PersistentFlags().StringVarP(&includeArg, "include", "I", "", "Include a list of paths") diff --git a/commands/command_migrate_import.go b/commands/command_migrate_import.go index b9ac784b..abdd538d 100644 --- a/commands/command_migrate_import.go +++ b/commands/command_migrate_import.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" + "github.com/git-lfs/git-lfs/errors" "github.com/git-lfs/git-lfs/filepathfilter" "github.com/git-lfs/git-lfs/git" "github.com/git-lfs/git-lfs/git/githistory" @@ -29,6 +30,70 @@ func migrateImportCommand(cmd *cobra.Command, args []string) { } defer db.Close() + if migrateNoRewrite { + if len(args) == 0 { + ExitWithError(errors.Errorf("fatal: expected one or more files with --no-rewrite")) + } + + ref, err := git.CurrentRef() + if err != nil { + ExitWithError(errors.Wrap(err, "fatal: unable to find current reference")) + } + + sha, _ := hex.DecodeString(ref.Sha) + commit, err := db.Commit(sha) + if err != nil { + ExitWithError(errors.Wrap(err, "fatal: unable to load commit")) + } + + root := commit.TreeID + + filter := git.GetAttributeFilter(cfg.LocalWorkingDir(), cfg.LocalGitDir()) + if len(filter.Include()) == 0 { + ExitWithError(errors.Errorf("fatal: no Git LFS filters found in .gitattributes")) + } + + gf := lfs.NewGitFilter(cfg) + + for _, file := range args { + if !filter.Allows(file) { + ExitWithError(errors.Errorf("fatal: file %s did not match any Git LFS filters in .gitattributes", file)) + } + } + + for _, file := range args { + root, err = rewriteTree(gf, db, root, file) + if err != nil { + ExitWithError(errors.Wrapf(err, "fatal: could not rewrite %q", file)) + } + } + + name, email := cfg.CurrentCommitter() + author := fmt.Sprintf("%s <%s>", name, email) + + oid, err := db.WriteCommit(&odb.Commit{ + Author: author, + Committer: author, + ParentIDs: [][]byte{sha}, + Message: generateMigrateCommitMessage(cmd, strings.Join(args, ",")), + TreeID: root, + }) + + if err != nil { + ExitWithError(errors.Wrap(err, "fatal: unable to write commit")) + } + + if err := git.UpdateRef(ref, oid, "git lfs migrate import --no-rewrite"); err != nil { + ExitWithError(errors.Wrap(err, "fatal: unable to update ref")) + } + + if err := checkoutNonBare(l); err != nil { + ExitWithError(errors.Wrap(err, "fatal: could not checkout")) + } + + return + } + rewriter := getHistoryRewriter(cmd, db, l) tracked := trackedFromFilter(rewriter.Filter()) @@ -36,7 +101,8 @@ func migrateImportCommand(cmd *cobra.Command, args []string) { gitfilter := lfs.NewGitFilter(cfg) migrate(args, rewriter, l, &githistory.RewriteOptions{ - Verbose: migrateVerbose, + Verbose: migrateVerbose, + ObjectMapFilePath: objectMapFilePath, BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) { if filepath.Base(path) == ".gitattributes" { return b, nil @@ -70,6 +136,18 @@ func migrateImportCommand(cmd *cobra.Command, args []string) { // include set is the wildcard filepath // extensions of files tracked. ours = exts + + if ours.Cardinality() == 0 { + // If it is still the case that we have + // no patterns to track, that means that + // we are in a tree that does not + // require .gitattributes changes. + // + // We can return early to avoid + // comparing and saving an identical + // tree. + return t, nil + } } theirs, err := trackedFromAttrs(db, t) @@ -103,19 +181,35 @@ func migrateImportCommand(cmd *cobra.Command, args []string) { UpdateRefs: true, }) - // Only perform `git-checkout(1) -f` if the repository is - // non-bare. - if bare, _ := git.IsBare(); !bare { - t := l.Waiter("migrate: checkout") - err := git.Checkout("", nil, true) - t.Complete() - - if err != nil { - ExitWithError(err) - } + if err := checkoutNonBare(l); err != nil { + ExitWithError(errors.Wrap(err, "fatal: could not checkout")) } } +// generateMigrateCommitMessage generates a commit message used with +// --no-rewrite, using --message (if given) or generating one if it isn't. +func generateMigrateCommitMessage(cmd *cobra.Command, patterns string) string { + if cmd.Flag("message").Changed { + return migrateCommitMessage + } + return fmt.Sprintf("%s: convert to Git LFS", patterns) +} + +// checkoutNonBare forces a checkout of the current reference, so long as the +// repository is non-bare. +// +// It returns nil on success, and a non-nil error on failure. +func checkoutNonBare(l *tasklog.Logger) error { + if bare, _ := git.IsBare(); bare { + return nil + } + + t := l.Waiter("migrate: checkout") + defer t.Complete() + + return git.Checkout("", nil, true) +} + // trackedFromFilter returns an ordered set of strings where each entry is a // line in the .gitattributes file. It adds/removes the fiter/diff/merge=lfs // attributes based on patterns included/excldued in the given filter. @@ -123,11 +217,11 @@ func trackedFromFilter(filter *filepathfilter.Filter) *tools.OrderedSet { tracked := tools.NewOrderedSet() for _, include := range filter.Include() { - tracked.Add(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text", include)) + tracked.Add(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text", escapeAttrPattern(include))) } for _, exclude := range filter.Exclude() { - tracked.Add(fmt.Sprintf("%s text -filter -merge -diff", exclude)) + tracked.Add(fmt.Sprintf("%s text -filter -merge -diff", escapeAttrPattern(exclude))) } return tracked @@ -201,3 +295,96 @@ func trackedToBlob(db *odb.ObjectDatabase, patterns *tools.OrderedSet) ([]byte, Size: int64(attrs.Len()), }) } + +// rewriteTree replaces the blob at the provided path within the given tree with +// a git lfs pointer. It will recursively rewrite any subtrees along the path to the +// blob. +func rewriteTree(gf *lfs.GitFilter, db *odb.ObjectDatabase, root []byte, path string) ([]byte, error) { + tree, err := db.Tree(root) + if err != nil { + return nil, err + } + + splits := strings.SplitN(path, "/", 2) + + switch len(splits) { + case 1: + // The path points to an entry at the root of this tree, so it must be a blob. + // Try to replace this blob with a Git LFS pointer. + index := findEntry(tree, splits[0]) + if index < 0 { + return nil, errors.Errorf("unable to find entry %s in tree", splits[0]) + } + + blobEntry := tree.Entries[index] + blob, err := db.Blob(blobEntry.Oid) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + + if _, err := clean(gf, &buf, blob.Contents, blobEntry.Name, blob.Size); err != nil { + return nil, err + } + + newOid, err := db.WriteBlob(&odb.Blob{ + Contents: &buf, + Size: int64(buf.Len()), + }) + + if err != nil { + return nil, err + } + + tree = tree.Merge(&odb.TreeEntry{ + Name: splits[0], + Filemode: blobEntry.Filemode, + Oid: newOid, + }) + return db.WriteTree(tree) + + case 2: + // The path points to an entry in a subtree contained at the root of the tree. + // Recursively rewrite the subtree. + head, tail := splits[0], splits[1] + + index := findEntry(tree, head) + if index < 0 { + return nil, errors.Errorf("unable to find entry %s in tree", head) + } + + subtreeEntry := tree.Entries[index] + if subtreeEntry.Type() != odb.TreeObjectType { + return nil, errors.Errorf("migrate: expected %s to be a tree, got %s", head, subtreeEntry.Type()) + } + + rewrittenSubtree, err := rewriteTree(gf, db, subtreeEntry.Oid, tail) + if err != nil { + return nil, err + } + + tree = tree.Merge(&odb.TreeEntry{ + Filemode: subtreeEntry.Filemode, + Name: subtreeEntry.Name, + Oid: rewrittenSubtree, + }) + + return db.WriteTree(tree) + + default: + return nil, errors.Errorf("error parsing path %s", path) + } +} + +// findEntry searches a tree for the desired entry, and returns the index of that +// entry within the tree's Entries array +func findEntry(t *odb.Tree, name string) int { + for i, entry := range t.Entries { + if entry.Name == name { + return i + } + } + + return -1 +} diff --git a/commands/command_status.go b/commands/command_status.go index 010b499b..47bec0ca 100644 --- a/commands/command_status.go +++ b/commands/command_status.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" "regexp" "strings" @@ -52,19 +53,29 @@ func statusCommand(cmd *cobra.Command, args []string) { ExitWithError(err) } + wd, _ := os.Getwd() + repo := cfg.LocalWorkingDir() + Print("\nGit LFS objects to be committed:\n") for _, entry := range staged { + // Find a path from the current working directory to the + // absolute path of each side of the entry. + src := relativize(wd, filepath.Join(repo, entry.SrcName)) + dst := relativize(wd, filepath.Join(repo, entry.DstName)) + switch entry.Status { case lfs.StatusRename, lfs.StatusCopy: - Print("\t%s -> %s (%s)", entry.SrcName, entry.DstName, formatBlobInfo(scanner, entry)) + Print("\t%s -> %s (%s)", src, dst, formatBlobInfo(scanner, entry)) default: - Print("\t%s (%s)", entry.SrcName, formatBlobInfo(scanner, entry)) + Print("\t%s (%s)", src, formatBlobInfo(scanner, entry)) } } Print("\nGit LFS objects not staged for commit:\n") for _, entry := range unstaged { - Print("\t%s (%s)", entry.SrcName, formatBlobInfo(scanner, entry)) + src := relativize(wd, filepath.Join(repo, entry.SrcName)) + + Print("\t%s (%s)", src, formatBlobInfo(scanner, entry)) } Print("") @@ -134,7 +145,7 @@ func blobInfo(s *lfs.PointerScanner, blobSha, name string) (sha, from string, er return s.ContentsSha()[:7], from, nil } - f, err := os.Open(name) + f, err := os.Open(filepath.Join(cfg.LocalWorkingDir(), name)) if err != nil { return "", "", err } @@ -311,6 +322,39 @@ func porcelainStatusLine(entry *lfs.DiffIndexEntry) string { return fmt.Sprintf("%s %s", entry.Status, entry.SrcName) } +// relativize relatives a path from "from" to "to". For instance, note that, for +// any paths "from" and "to", that: +// +// to == filepath.Clean(filepath.Join(from, relativize(from, to))) +func relativize(from, to string) string { + if len(from) == 0 { + return to + } + + flist := strings.Split(filepath.ToSlash(from), "/") + tlist := strings.Split(filepath.ToSlash(to), "/") + + var ( + divergence int + min int + ) + + if lf, lt := len(flist), len(tlist); lf < lt { + min = lf + } else { + min = lt + } + + for ; divergence < min; divergence++ { + if flist[divergence] != tlist[divergence] { + break + } + } + + return strings.Repeat("../", len(flist)-divergence) + + strings.Join(tlist[divergence:], "/") +} + func init() { RegisterCommand("status", statusCommand, func(cmd *cobra.Command) { cmd.Flags().BoolVarP(&porcelain, "porcelain", "p", false, "Give the output in an easy-to-parse format for scripts.") diff --git a/commands/command_track.go b/commands/command_track.go index 3472f9c3..8d160dcb 100644 --- a/commands/command_track.go +++ b/commands/command_track.go @@ -49,6 +49,8 @@ func trackCommand(cmd *cobra.Command, args []string) { return } + // Intentionally do _not_ consider global- and system-level + // .gitattributes here. knownPatterns := git.GetAttributePaths(cfg.LocalWorkingDir(), cfg.LocalGitDir()) lineEnd := getAttributeLineEnding(knownPatterns) if len(lineEnd) == 0 { @@ -67,7 +69,7 @@ func trackCommand(cmd *cobra.Command, args []string) { var writeablePatterns []string ArgsLoop: for _, unsanitizedPattern := range args { - pattern := cleanRootPath(unsanitizedPattern) + pattern := trimCurrentPrefix(cleanRootPath(unsanitizedPattern)) if !trackNoModifyAttrsFlag { for _, known := range knownPatterns { if known.Path == filepath.Join(relpath, pattern) && @@ -81,7 +83,7 @@ ArgsLoop: } // Generate the new / changed attrib line for merging - encodedArg := escapeTrackPattern(pattern) + encodedArg := escapeAttrPattern(pattern) lockableArg := "" if trackLockableFlag { // no need to test trackNotLockableFlag, if we got here we're disabling lockableArg = " " + git.LockableAttrib @@ -95,7 +97,7 @@ ArgsLoop: writeablePatterns = append(writeablePatterns, pattern) } - Print("Tracking %q", unescapeTrackPattern(encodedArg)) + Print("Tracking %q", unescapeAttrPattern(encodedArg)) } // Now read the whole local attributes file and iterate over the contents, @@ -213,7 +215,7 @@ ArgsLoop: } func listPatterns() { - knownPatterns := git.GetAttributePaths(cfg.LocalWorkingDir(), cfg.LocalGitDir()) + knownPatterns := getAllKnownPatterns() if len(knownPatterns) < 1 { return } @@ -228,6 +230,14 @@ func listPatterns() { } } +func getAllKnownPatterns() []git.AttributePath { + knownPatterns := git.GetAttributePaths(cfg.LocalWorkingDir(), cfg.LocalGitDir()) + knownPatterns = append(knownPatterns, git.GetRootAttributePaths(cfg.Git)...) + knownPatterns = append(knownPatterns, git.GetSystemAttributePaths(cfg.Os)...) + + return knownPatterns +} + func getAttributeLineEnding(attribs []git.AttributePath) string { for _, a := range attribs { if a.Source.Path == ".gitattributes" { @@ -258,7 +268,7 @@ var ( } ) -func escapeTrackPattern(unescaped string) string { +func escapeAttrPattern(unescaped string) string { var escaped string = strings.Replace(unescaped, `\`, "/", -1) for from, to := range trackEscapePatterns { @@ -268,7 +278,7 @@ func escapeTrackPattern(unescaped string) string { return escaped } -func unescapeTrackPattern(escaped string) string { +func unescapeAttrPattern(escaped string) string { var unescaped string = escaped for to, from := range trackEscapePatterns { diff --git a/commands/command_uninstall.go b/commands/command_uninstall.go index b27336cc..bdea8267 100644 --- a/commands/command_uninstall.go +++ b/commands/command_uninstall.go @@ -14,7 +14,9 @@ func uninstallCommand(cmd *cobra.Command, args []string) { uninstallHooksCommand(cmd, args) } - Print("Global Git LFS configuration has been removed.") + if !localInstall { + Print("Global Git LFS configuration has been removed.") + } } // uninstallHooksCmd removes any hooks created by Git LFS. diff --git a/commands/command_untrack.go b/commands/command_untrack.go index 0b51fca8..f15bfbb3 100644 --- a/commands/command_untrack.go +++ b/commands/command_untrack.go @@ -55,7 +55,7 @@ func untrackCommand(cmd *cobra.Command, args []string) { path := strings.Fields(line)[0] if removePath(path, args) { - Print("Untracking %q", unescapeTrackPattern(path)) + Print("Untracking %q", unescapeAttrPattern(path)) } else { attributesFile.WriteString(line + "\n") } @@ -63,8 +63,9 @@ func untrackCommand(cmd *cobra.Command, args []string) { } func removePath(path string, args []string) bool { + withoutCurrentDir := trimCurrentPrefix(path) for _, t := range args { - if path == escapeTrackPattern(t) { + if withoutCurrentDir == escapeAttrPattern(trimCurrentPrefix(t)) { return true } } diff --git a/commands/path.go b/commands/path.go index 1aa70833..4e7568cc 100644 --- a/commands/path.go +++ b/commands/path.go @@ -12,6 +12,24 @@ func gitLineEnding(git env) string { } } +const ( + windowsPrefix = `.\` + nixPrefix = `./` +) + +// trimCurrentPrefix removes a leading prefix of "./" or ".\" (referring to the +// current directory in a platform independent manner). +// +// It is useful for callers such as "git lfs track" and "git lfs untrack", that +// wish to compare filepaths and/or attributes patterns without cleaning across +// multiple platforms. +func trimCurrentPrefix(p string) string { + if strings.HasPrefix(p, windowsPrefix) { + return strings.TrimPrefix(p, windowsPrefix) + } + return strings.TrimPrefix(p, nixPrefix) +} + type env interface { Get(string) (string, bool) } diff --git a/commands/run.go b/commands/run.go index 6cc5faea..823e76b2 100644 --- a/commands/run.go +++ b/commands/run.go @@ -16,6 +16,8 @@ import ( var ( commandFuncs []func() *cobra.Command commandMu sync.Mutex + + rootVersion bool ) // NewCommand creates a new 'git-lfs' sub command, given a command name and @@ -49,7 +51,9 @@ func RegisterCommand(name string, runFn func(cmd *cobra.Command, args []string), // Run initializes the 'git-lfs' command and runs it with the given stdin and // command line args. -func Run() { +// +// It returns an exit code. +func Run() int { log.SetOutput(ErrorWriter) root := NewCommand("git-lfs", gitlfsCommand) @@ -60,6 +64,8 @@ func Run() { root.SetHelpFunc(helpCommand) root.SetUsageFunc(usageCommand) + root.Flags().BoolVarP(&rootVersion, "version", "v", false, "") + cfg = config.New() for _, f := range commandFuncs { @@ -68,13 +74,20 @@ func Run() { } } - root.Execute() + err := root.Execute() closeAPIClient() + + if err != nil { + return 127 + } + return 0 } func gitlfsCommand(cmd *cobra.Command, args []string) { versionCommand(cmd, args) - cmd.Usage() + if !rootVersion { + cmd.Usage() + } } func helpCommand(cmd *cobra.Command, args []string) { diff --git a/config/netrc.go b/config/netrc.go index 27156459..004755cd 100644 --- a/config/netrc.go +++ b/config/netrc.go @@ -4,7 +4,7 @@ import ( "os" "path/filepath" - "github.com/bgentry/go-netrc/netrc" + "github.com/git-lfs/go-netrc/netrc" ) type netrcfinder interface { diff --git a/debian/rules b/debian/rules index ddae758a..a20376d7 100755 --- a/debian/rules +++ b/debian/rules @@ -32,6 +32,7 @@ override_dh_clean: dh_clean override_dh_auto_build: + cd ${BUILD_DIR}/src/github.com/git-lfs/git-lfs && go generate ./commands dh_auto_build #dh_golang doesn't do anything here in deb 8, and it's needed in both if [ "$(DEB_HOST_GNU_TYPE)" != "$(DEB_BUILD_GNU_TYPE)" ]; then\ diff --git a/docs/man/git-lfs-checkout.1.ronn b/docs/man/git-lfs-checkout.1.ronn index c7dd82f3..adab6a91 100644 --- a/docs/man/git-lfs-checkout.1.ronn +++ b/docs/man/git-lfs-checkout.1.ronn @@ -7,6 +7,8 @@ git-lfs-checkout(1) -- Update working copy with file content if available ## DESCRIPTION +This command is deprecated, and should be replaced with `git checkout`. + Try to ensure that the working copy contains file content for Git LFS objects for the current ref, if the object data is available. Does not download any content, see git-lfs-fetch(1) for that. diff --git a/docs/man/git-lfs-migrate.1.ronn b/docs/man/git-lfs-migrate.1.ronn index 7ee3782a..9628156d 100644 --- a/docs/man/git-lfs-migrate.1.ronn +++ b/docs/man/git-lfs-migrate.1.ronn @@ -80,12 +80,48 @@ options and these additional ones: * `--verbose` Print the commit oid and filename of migrated files to STDOUT. -If `--include` or `--exclude` (`-I`, `-X`, respectively) are given, the -.gitattributes will be modified to include any new filepath patterns as given by -those flags. +* `--object-map=` + Write to 'path' a file with the mapping of each rewritten commits. The file + format is CSV with this pattern: `OLD-SHA`,`NEW-SHA` -If neither of those flags are given, the gitattributes will be incrementally -modified to include new filepath extensions as they are rewritten in history. +* `--no-rewrite` + Migrate large objects to Git LFS in a new commit without rewriting git + history. Please note that when this option is used, the `migrate import` + command will expect a different argument list, specialized options will + become available, and the core `migrate` options will be ignored. See + [IMPORT (NO REWRITE)]. + +If `--no-rewrite` is not provided and `--include` or `--exclude` (`-I`, `-X`, +respectively) are given, the .gitattributes will be modified to include any new +filepath patterns as given by those flags. + +If `--no-rewrite` is not provided and neither of those flags are given, the +gitattributes will be incrementally modified to include new filepath extensions +as they are rewritten in history. + +### IMPORT (NO REWRITE) + +The `import` mode has a special sub-mode enabled by the `--no-rewrite` flag. +This sub-mode will migrate large objects to pointers as in the base `import` +mode, but will do so in a new commit without rewriting Git history. When using +this sub-mode, the base `migrate` options, such as `--include-ref`, will be +ignored, as will those for the base `import` mode. The `migrate` command will +also take a different argument list. As a result of these changes, +`--no-rewrite` will only operate on the current branch - any other interested +branches must have the generated commit merged in. + +The `--no-rewrite` sub-mode supports the following options and arguments: + +* `-m --message=` + Specifies a commit message for the newly created commit. + +* [file ...] + The list of files to import. These files must be tracked by patterns + specified in the gitattributes. + +If `--message` is given, the new commit will be created with the provided +message. If no message is given, a commit message will be generated based on the +file arguments. ## INCLUDE AND EXCLUDE @@ -185,6 +221,20 @@ $ git lfs migrate import --everything --include="*.zip" Note: This will require a force push to any existing Git remotes. +### Migrate without rewriting local history + +You can also migrate files without modifying the existing history of your respoitory: + +Without a specified commit message: +``` + git lfs migrate import --no-rewrite test.zip *.mp3 *.psd +``` +With a specified commit message: +``` + git lfs migrate import --no-rewrite -m "Import .zip, .mp3, .psd files" \ + test.zip *.mpd *.psd +``` + ## SEE ALSO Part of the git-lfs(1) suite. diff --git a/docs/man/git-lfs-track.1.ronn b/docs/man/git-lfs-track.1.ronn index 234880e6..41810bca 100644 --- a/docs/man/git-lfs-track.1.ronn +++ b/docs/man/git-lfs-track.1.ronn @@ -59,6 +59,6 @@ to match paths. ## SEE ALSO -git-lfs-untrack(1), git-lfs-install(1), gitattributes(5). +git-lfs-untrack(1), git-lfs-install(1), gitattributes(5), gitignore(5). Part of the git-lfs(1) suite. diff --git a/docs/man/git-lfs.1.ronn b/docs/man/git-lfs.1.ronn index dc5cc42e..1621cfea 100644 --- a/docs/man/git-lfs.1.ronn +++ b/docs/man/git-lfs.1.ronn @@ -33,7 +33,7 @@ commands and low level ("plumbing") commands. Display the Git LFS environment. * git-lfs-checkout(1): Populate working copy with real content from Git LFS files. -* git lfs clone: +* git-lfs-clone(1): Efficiently clone a Git LFS-enabled repository. * git-lfs-fetch(1): Download Git LFS files from a remote. @@ -46,15 +46,16 @@ commands and low level ("plumbing") commands. * git-lfs-locks(1): List currently "locked" files from the Git LFS server. * git-lfs-logs(1): - Show errors from the git-lfs command. + Show errors from the Git LFS command. * git-lfs-ls-files(1): Show information about Git LFS files in the index and working tree. * git-lfs-migrate(1): - Migrate history to or from git-lfs + Migrate history to or from Git LFS * git-lfs-prune(1): Delete old Git LFS files from local storage * git-lfs-pull(1): - Fetch LFS changes from the remote & checkout any required working tree files. + Fetch Git LFS changes from the remote & checkout any required working tree + files. * git-lfs-push(1): Push queued large files to the Git LFS endpoint. * git-lfs-status(1): @@ -69,7 +70,7 @@ commands and low level ("plumbing") commands. Remove Git LFS paths from Git Attributes. * git-lfs-update(1): Update Git hooks for the current Git repository. -* git lfs version: +* git-lfs-version(1): Report the version number. ### Low level commands (plumbing) @@ -80,5 +81,32 @@ commands and low level ("plumbing") commands. Build and compare pointers. * git-lfs-pre-push(1): Git pre-push hook implementation. +* git-lfs-filter-process(1): + Git process filter that converts between large files and pointers. * git-lfs-smudge(1): Git smudge filter that converts pointer in blobs to the actual content. + +## EXAMPLES + +To get started with Git LFS, the following commands can be used. + + 1. Setup Git LFS on your system. You only have to do this once per + repository per machine: + + git lfs install + + 2. Choose the type of files you want to track, for examples all `ISO` + images, with git-lfs-track(1): + + git lfs track "*.iso" + + 3. The above stores this information in gitattributes(5) files, so + that file need to be added to the repository: + + git add .gitattributes + + 3. Commit, push and work with the files normally: + + git add file.iso + git commit -m "Add disk image" + git push diff --git a/git-lfs.go b/git-lfs.go index c185eea0..d8d66b41 100644 --- a/git-lfs.go +++ b/git-lfs.go @@ -32,6 +32,7 @@ func main() { } }() - commands.Run() + code := commands.Run() once.Do(commands.Cleanup) + os.Exit(code) } diff --git a/git/attribs.go b/git/attribs.go index 067bf42d..6a033b55 100644 --- a/git/attribs.go +++ b/git/attribs.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/git-lfs/git-lfs/filepathfilter" "github.com/git-lfs/git-lfs/tools" "github.com/rubyist/tracerx" ) @@ -34,6 +35,36 @@ func (s *AttributeSource) String() string { return s.Path } +// GetRootAttributePaths beahves as GetRootAttributePaths, and loads information +// only from the global gitattributes file. +func GetRootAttributePaths(cfg Env) []AttributePath { + af, ok := cfg.Get("core.attributesfile") + if !ok { + return nil + } + + // The working directory for the root gitattributes file is blank. + return attrPaths(af, "") +} + +// GetSystemAttributePaths behaves as GetAttributePaths, and loads information +// only from the system gitattributes file, respecting the $PREFIX environment +// variable. +func GetSystemAttributePaths(env Env) []AttributePath { + prefix, _ := env.Get("PREFIX") + if len(prefix) == 0 { + prefix = string(filepath.Separator) + } + + path := filepath.Join(prefix, "etc", "gitattributes") + + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil + } + + return attrPaths(path, "") +} + // GetAttributePaths returns a list of entries in .gitattributes which are // configured with the filter=lfs attribute // workingDir is the root of the working copy @@ -42,61 +73,87 @@ func GetAttributePaths(workingDir, gitDir string) []AttributePath { paths := make([]AttributePath, 0) for _, path := range findAttributeFiles(workingDir, gitDir) { - attributes, err := os.Open(path) - if err != nil { - continue - } - - relfile, _ := filepath.Rel(workingDir, path) - reldir := filepath.Dir(relfile) - source := &AttributeSource{Path: relfile} - - le := &lineEndingSplitter{} - scanner := bufio.NewScanner(attributes) - scanner.Split(le.ScanLines) - - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - - if strings.HasPrefix(line, "#") { - continue - } - - // Check for filter=lfs (signifying that LFS is tracking - // this file) or "lockable", which indicates that the - // file is lockable (and may or may not be tracked by - // Git LFS). - if strings.Contains(line, "filter=lfs") || - strings.HasSuffix(line, "lockable") { - - fields := strings.Fields(line) - pattern := fields[0] - if len(reldir) > 0 { - pattern = filepath.Join(reldir, pattern) - } - // Find lockable flag in any position after pattern to avoid - // edge case of matching "lockable" to a file pattern - lockable := false - for _, f := range fields[1:] { - if f == LockableAttrib { - lockable = true - break - } - } - paths = append(paths, AttributePath{ - Path: pattern, - Source: source, - Lockable: lockable, - }) - } - } - - source.LineEnding = le.LineEnding() + paths = append(paths, attrPaths(path, workingDir)...) } return paths } +func attrPaths(path, workingDir string) []AttributePath { + attributes, err := os.Open(path) + if err != nil { + return nil + } + + var paths []AttributePath + + relfile, _ := filepath.Rel(workingDir, path) + reldir := filepath.Dir(relfile) + source := &AttributeSource{Path: relfile} + + le := &lineEndingSplitter{} + scanner := bufio.NewScanner(attributes) + scanner.Split(le.ScanLines) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + if strings.HasPrefix(line, "#") { + continue + } + + // Check for filter=lfs (signifying that LFS is tracking + // this file) or "lockable", which indicates that the + // file is lockable (and may or may not be tracked by + // Git LFS). + if strings.Contains(line, "filter=lfs") || + strings.HasSuffix(line, "lockable") { + + fields := strings.Fields(line) + pattern := fields[0] + if len(reldir) > 0 { + pattern = filepath.Join(reldir, pattern) + } + // Find lockable flag in any position after pattern to avoid + // edge case of matching "lockable" to a file pattern + lockable := false + for _, f := range fields[1:] { + if f == LockableAttrib { + lockable = true + break + } + } + paths = append(paths, AttributePath{ + Path: pattern, + Source: source, + Lockable: lockable, + }) + } + } + + source.LineEnding = le.LineEnding() + + return paths +} + +// GetAttributeFilter returns a list of entries in .gitattributes which are +// configured with the filter=lfs attribute as a file path filter which +// file paths can be matched against +// workingDir is the root of the working copy +// gitDir is the root of the git repo +func GetAttributeFilter(workingDir, gitDir string) *filepathfilter.Filter { + paths := GetAttributePaths(workingDir, gitDir) + patterns := make([]filepathfilter.Pattern, 0, len(paths)) + + for _, path := range paths { + // Convert all separators to `/` before creating a pattern to + // avoid characters being escaped in situations like `subtree\*.md` + patterns = append(patterns, filepathfilter.NewPattern(filepath.ToSlash(path.Path))) + } + + return filepathfilter.NewFromPatterns(patterns, nil) +} + // copies bufio.ScanLines(), counting LF vs CRLF in a file type lineEndingSplitter struct { LFCount int diff --git a/git/git.go b/git/git.go index bd5b931f..51507ba6 100644 --- a/git/git.go +++ b/git/git.go @@ -245,7 +245,8 @@ func ResolveRef(ref string) (*Ref, error) { if len(lines) == 1 { // ref is a sha1 and has no symbolic-full-name - fullref.Name = lines[0] // fullref.Sha + fullref.Name = lines[0] + fullref.Sha = lines[0] fullref.Type = RefTypeOther return fullref, nil } @@ -974,7 +975,13 @@ func Fetch(remotes ...string) error { return nil } - _, err := gitNoLFSSimple(append([]string{"fetch"}, remotes...)...) + var args []string + if len(remotes) > 1 { + args = []string{"--multiple", "--"} + } + args = append(args, remotes...) + + _, err := gitNoLFSSimple(append([]string{"fetch"}, args...)...) return err } diff --git a/git/githistory/rewriter.go b/git/githistory/rewriter.go index 566e57e9..dd545f08 100644 --- a/git/githistory/rewriter.go +++ b/git/githistory/rewriter.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "fmt" "io" + "os" "strings" "sync" @@ -54,6 +55,10 @@ type RewriteOptions struct { // Verbose mode prints migrated objects. Verbose bool + // ObjectMapFilePath is the path to the map of old sha1 to new sha1 + // commits + ObjectMapFilePath string + // BlobFn specifies a function to rewrite blobs. // // It is called once per unique, unchanged path. That is to say, if @@ -188,6 +193,15 @@ func (r *Rewriter) Rewrite(opt *RewriteOptions) ([]byte, error) { vPerc = perc } + var objectMapFile *os.File + if len(opt.ObjectMapFilePath) > 0 { + objectMapFile, err = os.OpenFile(opt.ObjectMapFilePath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + return nil, fmt.Errorf("Could not create object map file: %v", err) + } + defer objectMapFile.Close() + } + // Keep track of the last commit that we rewrote. Callers often want // this so that they can perform a git-update-ref(1). var tip []byte @@ -253,6 +267,11 @@ func (r *Rewriter) Rewrite(opt *RewriteOptions) ([]byte, error) { if err != nil { return nil, err } + if objectMapFile != nil { + if _, err := fmt.Fprintf(objectMapFile, "%x,%x\n", oid, newSha); err != nil { + return nil, err + } + } } // Cache that commit so that we can reassign children of this @@ -322,6 +341,12 @@ func (r *Rewriter) rewriteTree(commitOID []byte, treeOID []byte, path string, fn continue } + // If this is a symlink, skip it + if entry.Filemode == 0120000 { + entries = append(entries, copyEntry(entry)) + continue + } + if cached := r.uncacheEntry(entry); cached != nil { entries = append(entries, copyEntry(cached)) continue diff --git a/git/githistory/rewriter_test.go b/git/githistory/rewriter_test.go index 0d570baf..45f342a7 100644 --- a/git/githistory/rewriter_test.go +++ b/git/githistory/rewriter_test.go @@ -3,7 +3,6 @@ package githistory import ( "bytes" "encoding/hex" - "fmt" "io" "io/ioutil" "reflect" @@ -61,8 +60,8 @@ func TestRewriterRewritesHistory(t *testing.T) { // // 100644 blob e440e5c842586965a7fb77deda2eca68612b1f53 hello.txt - AssertCommitParent(t, db, hex.EncodeToString(tip), "911994ab82ce256433c1fa739dbbbc7142156289") - AssertCommitTree(t, db, "911994ab82ce256433c1fa739dbbbc7142156289", tree2) + AssertCommitParent(t, db, hex.EncodeToString(tip), "4aaa3f49ffeabbb874250fe13ffeb8c683aba650") + AssertCommitTree(t, db, "4aaa3f49ffeabbb874250fe13ffeb8c683aba650", tree2) AssertBlobContents(t, db, tree2, "hello.txt", "3") @@ -71,8 +70,8 @@ func TestRewriterRewritesHistory(t *testing.T) { // // 100644 blob d8263ee9860594d2806b0dfd1bfd17528b0ba2a4 hello.txt - AssertCommitParent(t, db, "911994ab82ce256433c1fa739dbbbc7142156289", "38679ebeba3403103196eb6272b326f96c928ace") - AssertCommitTree(t, db, "38679ebeba3403103196eb6272b326f96c928ace", tree3) + AssertCommitParent(t, db, "4aaa3f49ffeabbb874250fe13ffeb8c683aba650", "24a341e1ff75addc22e336a8d87f82ba56b86fcf") + AssertCommitTree(t, db, "24a341e1ff75addc22e336a8d87f82ba56b86fcf", tree3) AssertBlobContents(t, db, tree3, "hello.txt", "2") } @@ -112,14 +111,14 @@ func TestRewriterRewritesOctopusMerges(t *testing.T) { // parent 1fe2b9577d5610e8d8fb2c3030534036fb648393 // parent ca447959bdcd20253d69b227bcc7c2e1d3126d5c - AssertCommitParent(t, db, hex.EncodeToString(tip), "89ab88fb7e11a439299aa2aa77a5d98f6629b750") - AssertCommitParent(t, db, hex.EncodeToString(tip), "adf1e9085f9dd263c1bec399b995ccfa5d994721") + AssertCommitParent(t, db, hex.EncodeToString(tip), "1fe2b9577d5610e8d8fb2c3030534036fb648393") + AssertCommitParent(t, db, hex.EncodeToString(tip), "ca447959bdcd20253d69b227bcc7c2e1d3126d5c") // And each of those parents should contain the root commit as their own // parent: - AssertCommitParent(t, db, "89ab88fb7e11a439299aa2aa77a5d98f6629b750", "52daca68bcf750bb86289fd95f92f5b3bd202328") - AssertCommitParent(t, db, "adf1e9085f9dd263c1bec399b995ccfa5d994721", "52daca68bcf750bb86289fd95f92f5b3bd202328") + AssertCommitParent(t, db, "1fe2b9577d5610e8d8fb2c3030534036fb648393", "9237567f379b3c83ddf53ad9a2ae3755afb62a09") + AssertCommitParent(t, db, "ca447959bdcd20253d69b227bcc7c2e1d3126d5c", "9237567f379b3c83ddf53ad9a2ae3755afb62a09") } func TestRewriterVisitsPackedObjects(t *testing.T) { @@ -189,10 +188,6 @@ func TestRewriterVisitsUniqueEntriesWithIdenticalContents(t *testing.T) { tree := "bbbe0a7676523ae02234bfe874784ca2380c2d4b" - fmt.Println(hex.EncodeToString(tip)) - root, _ := db.Root() - fmt.Println(root) - AssertCommitTree(t, db, hex.EncodeToString(tip), tree) // After rewriting, the HEAD state of the repository should contain a @@ -278,8 +273,8 @@ func TestRewriterAllowsAdditionalTreeEntries(t *testing.T) { // 100644 blob d8263ee9860594d2806b0dfd1bfd17528b0ba2a4 hello.txt // 100644 blob 0f2287157f7cb0dd40498c7a92f74b6975fa2d57 extra.txt - AssertCommitParent(t, db, hex.EncodeToString(tip), "54ca0fdd5ee455d872ce4b4e379abe1c4cdc39b3") - AssertCommitTree(t, db, "54ca0fdd5ee455d872ce4b4e379abe1c4cdc39b3", tree2) + AssertCommitParent(t, db, hex.EncodeToString(tip), "45af5deb9a25bc4069b15c1f5bdccb0340978707") + AssertCommitTree(t, db, "45af5deb9a25bc4069b15c1f5bdccb0340978707", tree2) AssertBlobContents(t, db, tree2, "hello.txt", "2") AssertBlobContents(t, db, tree2, "extra.txt", "extra\n") @@ -290,8 +285,8 @@ func TestRewriterAllowsAdditionalTreeEntries(t *testing.T) { // 100644 blob 56a6051ca2b02b04ef92d5150c9ef600403cb1de hello.txt // 100644 blob 0f2287157f7cb0dd40498c7a92f74b6975fa2d57 extra.txt - AssertCommitParent(t, db, "54ca0fdd5ee455d872ce4b4e379abe1c4cdc39b3", "4c52196256c611d18ad718b9b68b3d54d0a6686d") - AssertCommitTree(t, db, "4c52196256c611d18ad718b9b68b3d54d0a6686d", tree3) + AssertCommitParent(t, db, "45af5deb9a25bc4069b15c1f5bdccb0340978707", "99f6bd7cd69b45494afed95b026f3e450de8304f") + AssertCommitTree(t, db, "99f6bd7cd69b45494afed95b026f3e450de8304f", tree3) AssertBlobContents(t, db, tree3, "hello.txt", "1") AssertBlobContents(t, db, tree3, "extra.txt", "extra\n") @@ -355,8 +350,8 @@ func TestHistoryRewriterUpdatesRefs(t *testing.T) { assert.Nil(t, err) c1 := hex.EncodeToString(tip) - c2 := "86f7ba8f02edaca4f980cdd584ea8899e18b840c" - c3 := "d73b8c1a294e2371b287d9b75dbed82328ad446e" + c2 := "66561fe3ae68651658e18e48053dcfe66a2e9da1" + c3 := "8268d8486c48024a871fa42fc487dbeabd6e3d86" AssertRef(t, db, "refs/heads/master", tip) @@ -374,3 +369,18 @@ func TestHistoryRewriterReturnsFilter(t *testing.T) { assert.Equal(t, expected, got, "git/githistory: expected Rewriter.Filter() to return same *filepathfilter.Filter instance") } + +// debug is meant to be called from a defer statement to aide in debugging a +// test failure among any in this file. +// +// Callers are expected to call it immediately after calling the Rewrite() +// function. +func debug(t *testing.T, db *odb.ObjectDatabase, tip []byte, err error) { + root, ok := db.Root() + + t.Log(strings.Repeat("*", 80)) + t.Logf("* root=%s, ok=%t\n", root, ok) + t.Logf("* tip=%x\n", tip) + t.Logf("* err=%s\n", err) + t.Log(strings.Repeat("*", 80)) +} diff --git a/git/odb/commit.go b/git/odb/commit.go index bd96aa0d..bfa22b86 100644 --- a/git/odb/commit.go +++ b/git/odb/commit.go @@ -185,7 +185,11 @@ func (c *Commit) Encode(to io.Writer) (n int, err error) { n = n + n3 } - n4, err := fmt.Fprintf(to, "\n%s", c.Message) + // c.Message is built from messageParts in the Decode() function. + // + // Since each entry in messageParts _does not_ contain its trailing LF, + // append an empty string to capture the final newline. + n4, err := fmt.Fprintf(to, "\n%s\n", c.Message) if err != nil { return n, err } diff --git a/git/odb/object_db_test.go b/git/odb/object_db_test.go index 46069a62..cf2edaa5 100644 --- a/git/odb/object_db_test.go +++ b/git/odb/object_db_test.go @@ -159,7 +159,7 @@ func TestWriteCommit(t *testing.T) { Message: "initial commit", }) - expected := "77a746376fdb591a44a4848b5ba308b2d3e2a90c" + expected := "fee8a35c2890cd6e0e28d24cc457fcecbd460962" assert.Nil(t, err) assert.Equal(t, expected, hex.EncodeToString(sha)) diff --git a/glide.lock b/glide.lock index a4cb1f48..0d6e5477 100644 --- a/glide.lock +++ b/glide.lock @@ -1,18 +1,18 @@ -hash: bad2138ca7787101a7a23af2464319cc580f4285e90c07d11eb9f90ad3bb9604 -updated: 2018-02-27T14:39:39.133796-08:00 +hash: 5d2fbd8be4931b982d29c6ac8df833f139b28ffdb44ca062948a2386e2096a4d +updated: 2018-05-25T13:01:03.220513-07:00 imports: - name: github.com/alexbrainman/sspi version: 4729b3d4d8581b2db83864d1018926e4154f9406 subpackages: - ntlm -- name: github.com/bgentry/go-netrc - version: 9fd32a8b3d3d3f9d43c341bfe098430e07609480 - subpackages: - - netrc - name: github.com/davecgh/go-spew version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d subpackages: - spew +- name: github.com/git-lfs/go-netrc + version: e0e9ca483a183481412e6f5a700ff20a36177503 + subpackages: + - netrc - name: github.com/git-lfs/wildmatch version: 8a0518641565a619e62a2738c7d4498fc345daf6 - name: github.com/inconshreveable/mousetrap diff --git a/glide.yaml b/glide.yaml index eee8a77a..5e7758d7 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,7 +1,7 @@ package: github.com/git-lfs/git-lfs import: -- package: github.com/bgentry/go-netrc - version: 9fd32a8b3d3d3f9d43c341bfe098430e07609480 +- package: github.com/git-lfs/go-netrc + version: e0e9ca483a183481412e6f5a700ff20a36177503 subpackages: - netrc - package: github.com/kr/pty diff --git a/lfs/attribute.go b/lfs/attribute.go index fee511ab..bbe6f521 100644 --- a/lfs/attribute.go +++ b/lfs/attribute.go @@ -56,7 +56,21 @@ func filterAttribute() *Attribute { "process": "git-lfs filter-process", "required": "true", }, - Upgradeables: upgradeables(), + Upgradeables: map[string][]string{ + "clean": []string{ + "git-lfs clean %f", + }, + "smudge": []string{ + "git-lfs smudge %f", + "git-lfs smudge --skip %f", + "git-lfs smudge --skip -- %f", + }, + "process": []string{ + "git-lfs filter", + "git-lfs filter --skip", + "git-lfs filter-process --skip", + }, + }, } } @@ -69,24 +83,20 @@ func skipSmudgeFilterAttribute() *Attribute { "process": "git-lfs filter-process --skip", "required": "true", }, - Upgradeables: upgradeables(), - } -} - -func upgradeables() map[string][]string { - return map[string][]string{ - "clean": []string{"git-lfs clean %f"}, - "smudge": []string{ - "git-lfs smudge %f", - "git-lfs smudge --skip %f", - "git-lfs smudge -- %f", - "git-lfs smudge --skip -- %f", - }, - "process": []string{ - "git-lfs filter", - "git-lfs filter --skip", - "git-lfs filter-process", - "git-lfs filter-process --skip", + Upgradeables: map[string][]string{ + "clean": []string{ + "git-lfs clean -- %f", + }, + "smudge": []string{ + "git-lfs smudge %f", + "git-lfs smudge --skip %f", + "git-lfs smudge -- %f", + }, + "process": []string{ + "git-lfs filter", + "git-lfs filter --skip", + "git-lfs filter-process", + }, }, } } diff --git a/lfsapi/auth.go b/lfsapi/auth.go index d4bcc4ad..7d53e49c 100644 --- a/lfsapi/auth.go +++ b/lfsapi/auth.go @@ -9,8 +9,8 @@ import ( "os" "strings" - "github.com/bgentry/go-netrc/netrc" "github.com/git-lfs/git-lfs/errors" + "github.com/git-lfs/go-netrc/netrc" "github.com/rubyist/tracerx" ) @@ -24,6 +24,10 @@ var ( // authentication from netrc or git's credential helpers if necessary, // supporting basic and ntlm authentication. func (c *Client) DoWithAuth(remote string, req *http.Request) (*http.Response, error) { + return c.doWithAuth(remote, req, nil) +} + +func (c *Client) doWithAuth(remote string, req *http.Request, via []*http.Request) (*http.Response, error) { req.Header = c.extraHeadersFor(req) apiEndpoint, access, credHelper, credsURL, creds, err := c.getCreds(remote, req) @@ -31,7 +35,7 @@ func (c *Client) DoWithAuth(remote string, req *http.Request) (*http.Response, e return nil, err } - res, err := c.doWithCreds(req, credHelper, creds, credsURL, access) + res, err := c.doWithCreds(req, credHelper, creds, credsURL, access, via) if err != nil { if errors.IsAuthError(err) { newAccess := getAuthAccess(res) @@ -45,6 +49,12 @@ func (c *Client) DoWithAuth(remote string, req *http.Request) (*http.Response, e req.Header.Del("Authorization") credHelper.Reject(creds) } + + // This case represents a rejected request that + // should have been authenticated but wasn't. Do + // not count this against our redirection + // maximum, so do not recur through doWithAuth + // and instead call DoWithAuth. return c.DoWithAuth(remote, req) } } @@ -57,11 +67,11 @@ func (c *Client) DoWithAuth(remote string, req *http.Request) (*http.Response, e return res, err } -func (c *Client) doWithCreds(req *http.Request, credHelper CredentialHelper, creds Creds, credsURL *url.URL, access Access) (*http.Response, error) { +func (c *Client) doWithCreds(req *http.Request, credHelper CredentialHelper, creds Creds, credsURL *url.URL, access Access, via []*http.Request) (*http.Response, error) { if access == NTLMAccess { return c.doWithNTLM(req, credHelper, creds, credsURL) } - return c.do(req) + return c.do(req, "", via) } // getCreds fills the authorization header for the given request if possible, @@ -270,6 +280,8 @@ func hasScheme(what string) bool { } func requestHasAuth(req *http.Request) bool { + // The "Authorization" string constant is safe, since we assume that all + // request headers have been canonicalized. if len(req.Header.Get("Authorization")) > 0 { return true } diff --git a/lfsapi/client.go b/lfsapi/client.go index 9cfd0e27..ede1f3b3 100644 --- a/lfsapi/client.go +++ b/lfsapi/client.go @@ -7,6 +7,7 @@ import ( "io" "net" "net/http" + "net/textproto" "net/url" "os" "regexp" @@ -27,7 +28,18 @@ var ( httpRE = regexp.MustCompile(`\Ahttps?://`) ) +var hintFileUrl = strings.TrimSpace(` +hint: The remote resolves to a file:// URL, which can only work with a +hint: standalone transfer agent. See section "Using a Custom Transfer Type +hint: without the API server" in custom-transfers.md for details. +`) + func (c *Client) NewRequest(method string, e Endpoint, suffix string, body interface{}) (*http.Request, error) { + if strings.HasPrefix(e.Url, "file://") { + // Initial `\n` to avoid overprinting `Downloading LFS...`. + fmt.Fprintf(os.Stderr, "\n%s\n", hintFileUrl) + } + sshRes, err := c.sshResolveWithRetries(e, method) if err != nil { return nil, err @@ -77,16 +89,16 @@ func joinURL(prefix, suffix string) string { func (c *Client) Do(req *http.Request) (*http.Response, error) { req.Header = c.extraHeadersFor(req) - return c.do(req) + return c.do(req, "", nil) } // do performs an *http.Request respecting redirects, and handles the response // as defined in c.handleResponse. Notably, it does not alter the headers for // the request argument in any way. -func (c *Client) do(req *http.Request) (*http.Response, error) { +func (c *Client) do(req *http.Request, remote string, via []*http.Request) (*http.Response, error) { req.Header.Set("User-Agent", UserAgent) - res, err := c.doWithRedirects(c.httpClient(req.Host), req, nil) + res, err := c.doWithRedirects(c.httpClient(req.Host), req, remote, via) if err != nil { return res, err } @@ -153,13 +165,20 @@ func (c *Client) extraHeaders(u *url.URL) map[string][]string { } k, v := parts[0], strings.TrimSpace(parts[1]) + // If header keys are given in non-canonicalized form (e.g., + // "AUTHORIZATION" as opposed to "Authorization") they will not + // be returned in calls to net/http.Header.Get(). + // + // So, we avoid this problem by first canonicalizing header keys + // for extra headers. + k = textproto.CanonicalMIMEHeaderKey(k) m[k] = append(m[k], v) } return m } -func (c *Client) doWithRedirects(cli *http.Client, req *http.Request, via []*http.Request) (*http.Response, error) { +func (c *Client) doWithRedirects(cli *http.Client, req *http.Request, remote string, via []*http.Request) (*http.Response, error) { tracedReq, err := c.traceRequest(req) if err != nil { return nil, err @@ -229,7 +248,14 @@ func (c *Client) doWithRedirects(cli *http.Client, req *http.Request, via []*htt return res, err } - return c.doWithRedirects(cli, redirectedReq, via) + if len(req.Header.Get("Authorization")) > 0 { + // If the original request was authenticated (noted by the + // presence of the Authorization header), then recur through + // doWithAuth, retaining the requests via but only after + // authenticating the redirected request. + return c.doWithAuth(remote, redirectedReq, via) + } + return c.doWithRedirects(cli, redirectedReq, remote, via) } func (c *Client) httpClient(host string) *http.Client { diff --git a/lfsapi/client_test.go b/lfsapi/client_test.go index e31a7e92..f5500f80 100644 --- a/lfsapi/client_test.go +++ b/lfsapi/client_test.go @@ -1,11 +1,13 @@ package lfsapi import ( + "encoding/base64" "encoding/json" "fmt" "net" "net/http" "net/http/httptest" + "strings" "sync/atomic" "testing" @@ -175,6 +177,80 @@ func TestClientRedirect(t *testing.T) { assert.EqualError(t, err, "lfsapi/client: refusing insecure redirect, https->http") } +func TestClientRedirectReauthenticate(t *testing.T) { + var srv1, srv2 *httptest.Server + var called1, called2 uint32 + var creds1, creds2 Creds + + srv1 = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddUint32(&called1, 1) + + if hdr := r.Header.Get("Authorization"); len(hdr) > 0 { + parts := strings.SplitN(hdr, " ", 2) + typ, b64 := parts[0], parts[1] + + auth, err := base64.URLEncoding.DecodeString(b64) + assert.Nil(t, err) + assert.Equal(t, "Basic", typ) + assert.Equal(t, "user1:pass1", string(auth)) + + http.Redirect(w, r, srv2.URL+r.URL.Path, http.StatusMovedPermanently) + return + } + w.WriteHeader(http.StatusUnauthorized) + })) + + srv2 = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddUint32(&called2, 1) + + parts := strings.SplitN(r.Header.Get("Authorization"), " ", 2) + typ, b64 := parts[0], parts[1] + + auth, err := base64.URLEncoding.DecodeString(b64) + assert.Nil(t, err) + assert.Equal(t, "Basic", typ) + assert.Equal(t, "user2:pass2", string(auth)) + })) + + // Change the URL of srv2 to make it appears as if it is a different + // host. + srv2.URL = strings.Replace(srv2.URL, "127.0.0.1", "0.0.0.0", 1) + + creds1 = Creds(map[string]string{ + "protocol": "http", + "host": strings.TrimPrefix(srv1.URL, "http://"), + + "username": "user1", + "password": "pass1", + }) + creds2 = Creds(map[string]string{ + "protocol": "http", + "host": strings.TrimPrefix(srv2.URL, "http://"), + + "username": "user2", + "password": "pass2", + }) + + defer srv1.Close() + defer srv2.Close() + + c, err := NewClient(NewContext(nil, nil, nil)) + creds := newCredentialCacher() + creds.Approve(creds1) + creds.Approve(creds2) + c.Credentials = creds + + req, err := http.NewRequest("GET", srv1.URL, nil) + require.Nil(t, err) + + _, err = c.DoWithAuth("", req) + assert.Nil(t, err) + + // called1 is 2 since LFS tries an unauthenticated request first + assert.EqualValues(t, 2, called1) + assert.EqualValues(t, 1, called2) +} + func TestNewClient(t *testing.T) { c, err := NewClient(NewContext(nil, nil, map[string]string{ "lfs.dialtimeout": "151", diff --git a/lfsapi/endpoint.go b/lfsapi/endpoint.go index 8e91c11d..7ecb1598 100644 --- a/lfsapi/endpoint.go +++ b/lfsapi/endpoint.go @@ -106,3 +106,10 @@ func endpointFromGitUrl(u *url.URL, e *endpointGitFinder) Endpoint { u.Scheme = e.gitProtocol return Endpoint{Url: u.String()} } + +func endpointFromLocalPath(path string) Endpoint { + if !strings.HasSuffix(path, ".git") { + path = fmt.Sprintf("%s/.git", path) + } + return Endpoint{Url: fmt.Sprintf("file://%s", path)} +} diff --git a/lfsapi/endpoint_finder.go b/lfsapi/endpoint_finder.go index 9c7f8783..84c4f2ef 100644 --- a/lfsapi/endpoint_finder.go +++ b/lfsapi/endpoint_finder.go @@ -170,6 +170,9 @@ func (e *endpointGitFinder) NewEndpointFromCloneURL(rawurl string) Endpoint { func (e *endpointGitFinder) NewEndpoint(rawurl string) Endpoint { rawurl = e.ReplaceUrlAlias(rawurl) + if strings.HasPrefix(rawurl, "/") { + return endpointFromLocalPath(rawurl) + } u, err := url.Parse(rawurl) if err != nil { return endpointFromBareSshUrl(rawurl) diff --git a/lfsapi/endpoint_finder_test.go b/lfsapi/endpoint_finder_test.go index 0af7696f..3ff6f091 100644 --- a/lfsapi/endpoint_finder_test.go +++ b/lfsapi/endpoint_finder_test.go @@ -250,6 +250,22 @@ func TestBareGitEndpointAddsLfsSuffix(t *testing.T) { assert.Equal(t, "", e.SshPort) } +func TestLocalPathEndpointAddsDotGitDir(t *testing.T) { + finder := NewEndpointFinder(NewContext(nil, nil, map[string]string{ + "remote.origin.url": "/local/path", + })) + e := finder.Endpoint("download", "") + assert.Equal(t, "file:///local/path/.git/info/lfs", e.Url) +} + +func TestLocalPathEndpointPreservesDotGit(t *testing.T) { + finder := NewEndpointFinder(NewContext(nil, nil, map[string]string{ + "remote.origin.url": "/local/path.git", + })) + e := finder.Endpoint("download", "") + assert.Equal(t, "file:///local/path.git/info/lfs", e.Url) +} + func TestAccessConfig(t *testing.T) { type accessTest struct { Access string diff --git a/lfsapi/netrc.go b/lfsapi/netrc.go index f40f641a..3d2c6555 100644 --- a/lfsapi/netrc.go +++ b/lfsapi/netrc.go @@ -4,8 +4,8 @@ import ( "os" "path/filepath" - "github.com/bgentry/go-netrc/netrc" "github.com/git-lfs/git-lfs/config" + "github.com/git-lfs/go-netrc/netrc" ) type NetrcFinder interface { diff --git a/lfsapi/netrc_test.go b/lfsapi/netrc_test.go index f07e7f01..88f495dc 100644 --- a/lfsapi/netrc_test.go +++ b/lfsapi/netrc_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/bgentry/go-netrc/netrc" + "github.com/git-lfs/go-netrc/netrc" ) func TestNetrcWithHostAndPort(t *testing.T) { diff --git a/lfsapi/ntlm.go b/lfsapi/ntlm.go index 10b1ae7c..0170bb93 100644 --- a/lfsapi/ntlm.go +++ b/lfsapi/ntlm.go @@ -19,7 +19,7 @@ type ntmlCredentials struct { } func (c *Client) doWithNTLM(req *http.Request, credHelper CredentialHelper, creds Creds, credsURL *url.URL) (*http.Response, error) { - res, err := c.do(req) + res, err := c.do(req, "", nil) if err != nil && !errors.IsAuthError(err) { return res, err } @@ -86,7 +86,7 @@ func (c *Client) ntlmSendMessage(req *http.Request, message []byte) (*http.Respo msg := base64.StdEncoding.EncodeToString(message) req.Header.Set("Authorization", "NTLM "+msg) - return c.do(req) + return c.do(req, "", nil) } func parseChallengeResponse(res *http.Response) ([]byte, error) { diff --git a/lfsapi/proxy.go b/lfsapi/proxy.go index d67f2dcd..c7410774 100644 --- a/lfsapi/proxy.go +++ b/lfsapi/proxy.go @@ -49,13 +49,6 @@ func proxyFromClient(c *Client) func(req *http.Request) (*url.URL, error) { } func getProxyServers(u *url.URL, urlCfg *config.URLConfig, osEnv config.Environment) (httpsProxy string, httpProxy string, noProxy string) { - if urlCfg != nil { - httpProxy, _ = urlCfg.Get("http", u.String(), "proxy") - if strings.HasPrefix(httpProxy, "https://") { - httpsProxy = httpProxy - } - } - if osEnv == nil { return } @@ -76,6 +69,16 @@ func getProxyServers(u *url.URL, urlCfg *config.URLConfig, osEnv config.Environm httpProxy, _ = osEnv.Get("http_proxy") } + if urlCfg != nil { + gitProxy, ok := urlCfg.Get("http", u.String(), "proxy") + if len(gitProxy) > 0 && ok { + if strings.HasPrefix(gitProxy, "https://") { + httpsProxy = gitProxy + } + httpProxy = gitProxy + } + } + noProxy, _ = osEnv.Get("NO_PROXY") if len(noProxy) == 0 { noProxy, _ = osEnv.Get("no_proxy") diff --git a/lfsapi/proxy_test.go b/lfsapi/proxy_test.go index ad98419b..e8e7e76a 100644 --- a/lfsapi/proxy_test.go +++ b/lfsapi/proxy_test.go @@ -20,7 +20,7 @@ func TestHttpsProxyFromGitConfig(t *testing.T) { require.Nil(t, err) proxyURL, err := proxyFromClient(c)(req) - assert.Equal(t, "proxy-from-git-config:8080", proxyURL.Host) + assert.Equal(t, "proxy-from-env:8080", proxyURL.Host) assert.Nil(t, err) } diff --git a/script/packagecloud.rb b/script/packagecloud.rb index 6a57347a..1b8b1382 100644 --- a/script/packagecloud.rb +++ b/script/packagecloud.rb @@ -62,6 +62,7 @@ $distro_name_map = { ubuntu/yakkety ubuntu/zesty ubuntu/artful + ubuntu/bionic ), } diff --git a/test/test-checkout.sh b/test/test-checkout.sh index ef0ee6ae..cd2633c8 100755 --- a/test/test-checkout.sh +++ b/test/test-checkout.sh @@ -41,12 +41,13 @@ begin_test "checkout" rm -rf file1.dat file2.dat file3.dat folder1/nested.dat folder2/nested.dat echo "checkout should replace all" - git lfs checkout + git lfs checkout 2>&1 | tee checkout.log [ "$contents" = "$(cat file1.dat)" ] [ "$contents" = "$(cat file2.dat)" ] [ "$contents" = "$(cat file3.dat)" ] [ "$contents" = "$(cat folder1/nested.dat)" ] [ "$contents" = "$(cat folder2/nested.dat)" ] + grep "Checking out LFS objects: 100% (5/5), 95 B" checkout.log # Remove the working directory rm -rf file1.dat file2.dat file3.dat folder1/nested.dat folder2/nested.dat @@ -73,7 +74,7 @@ begin_test "checkout" [ ! -f ../folder2/nested.dat ] # test '.' in current dir rm nested.dat - git lfs checkout . + git lfs checkout . 2>&1 | tee checkout.log [ "$contents" = "$(cat nested.dat)" ] popd diff --git a/test/test-credentials.sh b/test/test-credentials.sh index 3efbe6b5..845d484b 100755 --- a/test/test-credentials.sh +++ b/test/test-credentials.sh @@ -259,6 +259,39 @@ begin_test "credentials from netrc" ) end_test +begin_test "credentials from netrc with unknown keyword" +( + set -e + + printf "machine localhost\nlogin netrcuser\nnot-a-key something\npassword netrcpass\n" >> "$NETRCFILE" + echo $HOME + echo "GITSERVER $GITSERVER" + cat $NETRCFILE + + # prevent prompts on Windows particularly + export SSH_ASKPASS= + + reponame="netrctest" + setup_remote_repo "$reponame" + + clone_repo "$reponame" repo2 + + # Need a remote named "localhost" or 127.0.0.1 in netrc will interfere with the other auth + git remote add "netrc" "$(echo $GITSERVER | sed s/127.0.0.1/localhost/)/netrctest" + git lfs env + + git lfs track "*.dat" + echo "push a" > a.dat + git add .gitattributes a.dat + git commit -m "add a.dat" + + GIT_TRACE=1 git lfs push netrc master 2>&1 | tee push.log + grep "Uploading LFS objects: 100% (1/1), 7 B" push.log + echo "any git credential calls:" + [ "0" -eq "$(cat push.log | grep "git credential" | wc -l)" ] +) +end_test + begin_test "credentials from netrc with bad password" ( set -e @@ -274,7 +307,7 @@ begin_test "credentials from netrc with bad password" reponame="netrctest" setup_remote_repo "$reponame" - clone_repo "$reponame" repo2 + clone_repo "$reponame" repo3 # Need a remote named "localhost" or 127.0.0.1 in netrc will interfere with the other auth git remote add "netrc" "$(echo $GITSERVER | sed s/127.0.0.1/localhost/)/netrctest" diff --git a/test/test-env.sh b/test/test-env.sh index 5879bff0..2a3047e2 100755 --- a/test/test-env.sh +++ b/test/test-env.sh @@ -50,7 +50,7 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" ) @@ -102,12 +102,12 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$endpoint" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" cd .git expected2=$(echo "$expected" | sed -e 's/LocalWorkingDir=.*/LocalWorkingDir=/') - actual2=$(git lfs env) + actual2=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected2" "$actual2" ) end_test @@ -161,12 +161,12 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$endpoint" "$endpoint2" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" cd .git expected2=$(echo "$expected" | sed -e 's/LocalWorkingDir=.*/LocalWorkingDir=/') - actual2=$(git lfs env) + actual2=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected2" "$actual2" ) end_test @@ -218,12 +218,12 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$endpoint" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" cd .git expected2=$(echo "$expected" | sed -e 's/LocalWorkingDir=.*/LocalWorkingDir=/') - actual2=$(git lfs env) + actual2=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected2" "$actual2" ) end_test @@ -277,12 +277,12 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$endpoint" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" cd .git expected2=$(echo "$expected" | sed -e 's/LocalWorkingDir=.*/LocalWorkingDir=/') - actual2=$(git lfs env) + actual2=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected2" "$actual2" ) end_test @@ -337,12 +337,12 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" cd .git expected2=$(echo "$expected" | sed -e 's/LocalWorkingDir=.*/LocalWorkingDir=/') - actual2=$(git lfs env) + actual2=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected2" "$actual2" ) end_test @@ -398,12 +398,12 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" cd .git expected2=$(echo "$expected" | sed -e 's/LocalWorkingDir=.*/LocalWorkingDir=/') - actual2=$(git lfs env) + actual2=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected2" "$actual2" ) end_test @@ -466,12 +466,12 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" mkdir a cd a - actual2=$(git lfs env) + actual2=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual2" ) end_test @@ -522,19 +522,23 @@ UploadTransfers=basic %s ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(GIT_DIR=$gitDir GIT_WORK_TREE=$workTree git lfs env) + actual=$(GIT_DIR=$gitDir GIT_WORK_TREE=$workTree git lfs env \ + | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" cd $TRASHDIR/$reponame - actual2=$(GIT_DIR=$gitDir GIT_WORK_TREE=$workTree git lfs env) + actual2=$(GIT_DIR=$gitDir GIT_WORK_TREE=$workTree git lfs env \ + | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual2" cd $TRASHDIR/$reponame/.git - actual3=$(GIT_DIR=$gitDir GIT_WORK_TREE=$workTree git lfs env) + actual3=$(GIT_DIR=$gitDir GIT_WORK_TREE=$workTree git lfs env \ + | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual3" cd $TRASHDIR/$reponame/a/b/c - actual4=$(GIT_DIR=$gitDir GIT_WORK_TREE=$workTree git lfs env) + actual4=$(GIT_DIR=$gitDir GIT_WORK_TREE=$workTree git lfs env \ + | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual4" envVars="$(GIT_DIR=$gitDir GIT_WORK_TREE=a/b env | grep "^GIT" | sort)" @@ -565,7 +569,8 @@ DownloadTransfers=basic UploadTransfers=basic %s ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars") - actual5=$(GIT_DIR=$gitDir GIT_WORK_TREE=a/b git lfs env) + actual5=$(GIT_DIR=$gitDir GIT_WORK_TREE=a/b git lfs env \ + | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected5" "$actual5" cd $TRASHDIR/$reponame/a/b @@ -598,7 +603,7 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual7=$(GIT_DIR=$gitDir git lfs env) + actual7=$(GIT_DIR=$gitDir git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected7" "$actual7" cd $TRASHDIR/$reponame/a @@ -631,7 +636,7 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual8=$(GIT_WORK_TREE=$workTree git lfs env) + actual8=$(GIT_WORK_TREE=$workTree git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected8" "$actual8" ) end_test @@ -676,7 +681,7 @@ UploadTransfers=basic %s %s " "$(git lfs version)" "$(git version)" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" ) @@ -698,7 +703,8 @@ Endpoint (other)=https://other-git-server.com/user/repo.git/info/lfs (auth=none) SSH=git@other-git-server.com:user/repo.git GIT_SSH=lfs-ssh-echo' - contains_same_elements "$expected" "$(git lfs env | grep -e "Endpoint" -e "SSH=")" + contains_same_elements "$expected" "$(git lfs env \ + | grep -v "^GIT_EXEC_PATH=" | grep -e "Endpoint" -e "SSH=")" ) end_test @@ -754,7 +760,7 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expectedenabled" "$actual" git config --unset lfs.skipdownloaderrors @@ -787,11 +793,11 @@ UploadTransfers=basic %s %s ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expecteddisabled" "$actual" # now enable via env var - actual=$(GIT_LFS_SKIP_DOWNLOAD_ERRORS=1 git lfs env) + actual=$(GIT_LFS_SKIP_DOWNLOAD_ERRORS=1 git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expectedenabled" "$actual" @@ -853,7 +859,7 @@ UploadTransfers=basic,supertransfer,tus %s %s ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$lfsstorage" "$envVars" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expectedenabled" "$actual" ) diff --git a/test/test-extra-header.sh b/test/test-extra-header.sh index 1dbcfe30..b3a5f919 100755 --- a/test/test-extra-header.sh +++ b/test/test-extra-header.sh @@ -34,7 +34,7 @@ begin_test "http..extraHeader with authorization" setup_remote_repo "$reponame" clone_repo "$reponame" "$reponame" - # See: test/cmd/lfstest-gitserver.go:1176. + # See: test/cmd/lfstest-gitserver.go:missingRequiredCreds(). user="requirecreds" pass="pass" auth="Basic $(echo -n $user:$pass | base64)" @@ -59,3 +59,40 @@ begin_test "http..extraHeader with authorization" [ "0" -eq "$(grep -c "creds: git credential reject" curl.log)" ] ) end_test + +begin_test "http..extraHeader with authorization (casing)" +( + set -e + + reponame="requirecreds-extraHeaderCasing" + setup_remote_repo "$reponame" + clone_repo "$reponame" "$reponame" + + # See: test/cmd/lfstest-gitserver.go:missingRequiredCreds(). + user="requirecreds" + pass="pass" + auth="Basic $(echo -n $user:$pass | base64)" + + git config --local --add lfs.access basic + # N.B.: "AUTHORIZATION" is not the correct casing, and is therefore the + # subject of this test. See lfsapi.Client.extraHeaders() for more. + git config --local --add "http.extraHeader" "AUTHORIZATION: $auth" + + git lfs track "*.dat" + printf "contents" > a.dat + git add .gitattributes a.dat + git commit -m "initial commit" + + git push origin master 2>&1 | tee curl.log + if [ "0" -ne "${PIPESTATUS[0]}" ]; then + echo >&2 "expected \`git push origin master\` to succeed, didn't" + exit 1 + fi + + [ "0" -eq "$(grep -c "creds: filling with GIT_ASKPASS" curl.log)" ] + [ "0" -eq "$(grep -c "creds: git credential approve" curl.log)" ] + [ "0" -eq "$(grep -c "creds: git credential cache" curl.log)" ] + [ "0" -eq "$(grep -c "creds: git credential fill" curl.log)" ] + [ "0" -eq "$(grep -c "creds: git credential reject" curl.log)" ] +) +end_test diff --git a/test/test-install.sh b/test/test-install.sh index d4ef864d..abc8f641 100755 --- a/test/test-install.sh +++ b/test/test-install.sh @@ -4,17 +4,22 @@ begin_test "install again" ( - set -e + set -eo pipefail smudge="$(git config filter.lfs.smudge)" clean="$(git config filter.lfs.clean)" filter="$(git config filter.lfs.process)" - printf "$smudge" | grep "git-lfs smudge" - printf "$clean" | grep "git-lfs clean" - printf "$filter" | grep "git-lfs filter-process" + [ "$smudge" = "git-lfs smudge -- %f" ] + [ "$clean" = "git-lfs clean -- %f" ] + [ "$filter" = "git-lfs filter-process" ] - git lfs install + GIT_TRACE=1 git lfs install --skip-repo 2>&1 | tee install.log + + if grep -q "--replace-all" install.log; then + echo >&2 "fatal: unexpected git config --replace-all via 'git lfs install'" + exit 1 + fi [ "$smudge" = "$(git config filter.lfs.smudge)" ] [ "$clean" = "$(git config filter.lfs.clean)" ] diff --git a/test/test-lock.sh b/test/test-lock.sh index 5b90e501..bd82b4f0 100755 --- a/test/test-lock.sh +++ b/test/test-lock.sh @@ -12,7 +12,7 @@ begin_test "lock with good ref" git lfs lock "a.dat" --json 2>&1 | tee lock.json if [ "0" -ne "${PIPESTATUS[0]}" ]; then - echo >&2 "fatal: expected 'git lfs lock \'a.dat\'' to succeed" + echo >&2 "fatal: expected \'git lfs lock \'a.dat\'\' to succeed" exit 1 fi @@ -40,7 +40,7 @@ begin_test "lock with good tracked ref" git lfs lock "a.dat" --json 2>&1 | tee lock.json if [ "0" -ne "${PIPESTATUS[0]}" ]; then - echo >&2 "fatal: expected 'git lfs lock \'a.dat\'' to succeed" + echo >&2 "fatal: expected \'git lfs lock \'a.dat\'\' to succeed" exit 1 fi @@ -65,7 +65,7 @@ begin_test "lock with bad ref" GIT_CURL_VERBOSE=1 git lfs lock "a.dat" 2>&1 | tee lock.json if [ "0" -eq "${PIPESTATUS[0]}" ]; then - echo >&2 "fatal: expected 'git lfs lock \'a.dat\'' to fail" + echo >&2 "fatal: expected \'git lfs lock \'a.dat\'\' to fail" exit 1 fi @@ -187,7 +187,7 @@ begin_test "creating a lock (within subdirectory)" git lfs lock --json "a.dat" | tee lock.json if [ "0" -ne "${PIPESTATUS[0]}" ]; then - echo >&2 "fatal: expected 'git lfs lock \'a.dat\'' to succeed" + echo >&2 "fatal: expected \'git lfs lock \'a.dat\'\' to succeed" exit 1 fi @@ -195,3 +195,37 @@ begin_test "creating a lock (within subdirectory)" assert_server_lock "$reponame" "$id" ) end_test + +begin_test "creating a lock (symlinked working directory)" +( + set -eo pipefail + + if [[ $(uname) == *"MINGW"* ]]; then + echo >&2 "info: skipped on Windows ..." + exit 0 + fi + + reponame="lock-in-symlinked-working-directory" + setup_remote_repo "$reponame" + clone_repo "$reponame" "$reponame" + + git lfs track -l "*.dat" + mkdir -p folder1 folder2 + printf "hello" > folder2/a.dat + add_symlink "../folder2" "folder1/folder2" + + git add --all . + git commit -m "initial commit" + git push origin master + + pushd "$TRASHDIR" > /dev/null + ln -s "$reponame" "$reponame-symlink" + cd "$reponame-symlink" + + git lfs lock --json folder1/folder2/a.dat 2>&1 | tee lock.json + + id="$(assert_lock lock.json folder1/folder2/a.dat)" + assert_server_lock "$reponame" "$id" master + popd > /dev/null +) +end_test diff --git a/test/test-mergetool.sh b/test/test-mergetool.sh new file mode 100755 index 00000000..baac2e81 --- /dev/null +++ b/test/test-mergetool.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +. "test/testlib.sh" + +begin_test "mergetool works with large files" +( + set -e + + reponame="mergetool-works-with-large-files" + git init "$reponame" + cd "$reponame" + + git lfs track "*.dat" + printf "base" > conflict.dat + git add .gitattributes conflict.dat + git commit -m "initial commit" + + git checkout -b conflict + printf "b" > conflict.dat + git add conflict.dat + git commit -m "conflict.dat: b" + + git checkout master + + printf "a" > conflict.dat + git add conflict.dat + git commit -m "conflict.dat: a" + + set +e + git merge conflict + set -e + + git config mergetool.inspect.cmd ' + for i in BASE LOCAL REMOTE; do + echo "\$$i=$(eval "cat \"\$$i\"")"; + done; + exit 1 + ' + git config mergetool.inspect.trustExitCode true + + yes | git mergetool \ + --no-prompt \ + --tool=inspect \ + -- conflict.dat 2>&1 \ + | tee mergetool.log + + grep "\$BASE=base" mergetool.log + grep "\$LOCAL=a" mergetool.log + grep "\$REMOTE=b" mergetool.log +) +end_test diff --git a/test/test-migrate-fixtures.sh b/test/test-migrate-fixtures.sh index e461279c..5beee7d9 100755 --- a/test/test-migrate-fixtures.sh +++ b/test/test-migrate-fixtures.sh @@ -18,7 +18,7 @@ assert_ref_unmoved() { fi } -# setup_multiple_local_branches creates a repository as follows: +# setup_local_branch_with_gitattrs creates a repository as follows: # # A---B # \ @@ -44,6 +44,46 @@ setup_local_branch_with_gitattrs() { git commit -m "add .gitattributes" } +# setup_local_branch_with_nested_gitattrs creates a repository as follows: +# +# A---B +# \ +# refs/heads/master +# +# - Commit 'A' has 120, in a.txt, and a corresponding entry in .gitattributes. There is also +# 140 in a.md, with no corresponding entry in .gitattributes. +# It also has 140 in subtree/a.md, and a corresponding entry in subtree/.gitattributes +setup_local_branch_with_nested_gitattrs() { + set -e + + reponame="nested-attrs" + + remove_and_create_local_repo "$reponame" + + mkdir b + + base64 < /dev/urandom | head -c 120 > a.txt + base64 < /dev/urandom | head -c 140 > a.md + base64 < /dev/urandom | head -c 140 > b/a.md + + git add a.txt a.md b/a.md + git commit -m "initial commit" + + git lfs track "*.txt" + + git add .gitattributes + git commit -m "add .gitattributes" + + cd b + + git lfs track "*.md" + + cd .. + + git add b/.gitattributes + git commit -m "add nested .gitattributes" +} + # setup_multiple_local_branches creates a repository as follows: # # B @@ -79,6 +119,42 @@ setup_multiple_local_branches() { git checkout master } +# setup_multiple_local_branches_with_gitattrs creates a repository in the same way +# as setup_multiple_local_branches, but also adds relevant lfs filters to the +# .gitattributes file in the master branch +setup_multiple_local_branches_with_gitattrs() { + set -e + + setup_multiple_local_branches + + git lfs track *.txt + git lfs track *.md + + git add .gitattributes + git commit -m "add .gitattributes" +} + +# setup_local_branch_with_space creates a repository as follows: +# +# A +# \ +# refs/heads/master +# +# - Commit 'A' has 50 bytes in a file named "a file.txt". +setup_local_branch_with_space() { + set -e + + reponame="migrate-local-branch-with-space" + filename="a file.txt" + + remove_and_create_local_repo "$reponame" + + base64 < /dev/urandom | head -c 50 > "$filename" + + git add "$filename" + git commit -m "initial commit" +} + # setup_single_remote_branch creates a repository as follows: # # A---B @@ -113,6 +189,18 @@ setup_single_remote_branch() { git commit -m "add an additional 30, 50 bytes to a.{txt,md}" } +setup_single_remote_branch_with_gitattrs() { + set -e + + setup_single_remote_branch + + git lfs track *.txt + git lfs track *.md + + git add .gitattributes + git commit -m "add .gitattributes" +} + # setup_multiple_remote_branches creates a repository as follows: # # C @@ -220,6 +308,31 @@ setup_single_local_branch_with_annotated_tags() { git tag "v1.0.0" -m "v1.0.0" } +setup_multiple_remotes() { + set -e + + reponame="migrate-multiple-remotes" + remove_and_create_remote_repo "$reponame" + + forkname="$(git remote -v \ + | head -n1 \ + | cut -d ' ' -f 1 \ + | sed -e 's/^.*\///g')-fork" + ( setup_remote_repo "$forkname" ) + + git remote add fork "$GITSERVER/$forkname" + + base64 < /dev/urandom | head -c 16 > a.txt + git add a.txt + git commit -m "initial commit" + git push origin master + + base64 < /dev/urandom | head -c 16 > a.txt + git add a.txt + git commit -m "another commit" + git push fork master +} + # setup_single_local_branch_deep_trees creates a repository as follows: # # A @@ -240,6 +353,29 @@ setup_single_local_branch_deep_trees() { git commit -m "initial commit" } +# setup_local_branch_with_symlink creates a repository as follows: +# +# A +# \ +# refs/heads/master +# +# - Commit 'A' has 120, in a.txt, and a symbolic link link.txt to a.txt. +setup_local_branch_with_symlink() { + set -e + + reponame="migrate-single-local-branch-with-symlink" + + remove_and_create_local_repo "$reponame" + + base64 < /dev/urandom | head -c 120 > a.txt + + git add a.txt + git commit -m "initial commit" + + add_symlink "a.txt" "link.txt" + git commit -m "add symlink" +} + # make_bare converts the existing full checkout of a repository into a bare one, # and then `cd`'s into it. make_bare() { diff --git a/test/test-migrate-import-no-rewrite.sh b/test/test-migrate-import-no-rewrite.sh new file mode 100755 index 00000000..2bae29f8 --- /dev/null +++ b/test/test-migrate-import-no-rewrite.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash + +. "test/test-migrate-fixtures.sh" +. "test/testlib.sh" + +begin_test "migrate import --no-rewrite (default branch)" +( + set -e + + setup_local_branch_with_gitattrs + + txt_oid="$(calc_oid "$(git cat-file -p :a.txt)")" + prev_commit_oid="$(git rev-parse HEAD)" + + git lfs migrate import --no-rewrite *.txt + + # Ensure our desired files were imported into git-lfs + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "120" + assert_local_object "$txt_oid" "120" + + # Ensure the git history remained the same + new_commit_oid="$(git rev-parse HEAD~1)" + if [ "$prev_commit_oid" != "$new_commit_oid" ]; then + exit 1 + fi + + # Ensure a new commit was made + new_head_oid="$(git rev-parse HEAD)" + if [ "$prev_commit_oid" == "$new_oid" ]; then + exit 1 + fi + + # Ensure a new commit message was generated based on the list of imported files + commit_msg="$(git log -1 --pretty=format:%s)" + echo "$commit_msg" | grep -q "a.txt: convert to Git LFS" +) +end_test + +begin_test "migrate import --no-rewrite (bare repository)" +( + set -e + + setup_single_remote_branch_with_gitattrs + + prev_commit_oid="$(git rev-parse HEAD)" + txt_oid="$(calc_oid "$(git cat-file -p :a.txt)")" + md_oid="$(calc_oid "$(git cat-file -p :a.md)")" + + git lfs migrate import --no-rewrite a.txt a.md + + # Ensure our desired files were imported + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "30" + assert_pointer "refs/heads/master" "a.md" "$md_oid" "50" + + # Ensure the git history remained the same + new_commit_oid="$(git rev-parse HEAD~1)" + if [ "$prev_commit_oid" != "$new_commit_oid" ]; then + exit 1 + fi + + # Ensure a new commit was made + new_head_oid="$(git rev-parse HEAD)" + if [ "$prev_commit_oid" == "$new_oid" ]; then + exit 1 + fi +) +end_test + +begin_test "migrate import --no-rewrite (multiple branches)" +( + set -e + + setup_multiple_local_branches_with_gitattrs + + prev_commit_oid="$(git rev-parse HEAD)" + + md_oid="$(calc_oid "$(git cat-file -p :a.md)")" + txt_oid="$(calc_oid "$(git cat-file -p :a.txt)")" + md_feature_oid="$(calc_oid "$(git cat-file -p my-feature:a.md)")" + + git lfs migrate import --no-rewrite *.txt *.md + + # Ensure our desired files were imported + assert_pointer "refs/heads/master" "a.md" "$md_oid" "140" + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "120" + + assert_local_object "$md_oid" "140" + assert_local_object "$txt_oid" "120" + + # Ensure our other branch was unmodified + refute_local_object "$md_feature_oid" "30" + + # Ensure the git history remained the same + new_commit_oid="$(git rev-parse HEAD~1)" + if [ "$prev_commit_oid" != "$new_commit_oid" ]; then + exit 1 + fi + + # Ensure a new commit was made + new_head_oid="$(git rev-parse HEAD)" + if [ "$prev_commit_oid" == "$new_oid" ]; then + exit 1 + fi +) +end_test + +begin_test "migrate import --no-rewrite (no .gitattributes)" +( + set -e + + setup_multiple_local_branches + + # Ensure command fails if no .gitattributes files are present + git lfs migrate import --no-rewrite *.txt *.md 2>&1 | tee migrate.log + if [ ${PIPESTATUS[0]} -eq 0 ]; then + echo >&2 "fatal: expected git lfs migrate import --no-rewrite to fail, didn't" + exit 1 + fi + + grep "no Git LFS filters found in .gitattributes" migrate.log +) +end_test + +begin_test "migrate import --no-rewrite (nested .gitattributes)" +( + set -e + + setup_local_branch_with_nested_gitattrs + + # Ensure a .md filter does not exist in the top-level .gitattributes + master_attrs="$(git cat-file -p "$master:.gitattributes")" + [ !"$(echo "$master_attrs" | grep -q ".md")" ] + + # Ensure a .md filter exists in the nested .gitattributes + nested_attrs="$(git cat-file -p "$master:b/.gitattributes")" + echo "$nested_attrs" | grep -q "*.md filter=lfs diff=lfs merge=lfs" + + md_oid="$(calc_oid "$(git cat-file -p :a.md)")" + nested_md_oid="$(calc_oid "$(git cat-file -p :b/a.md)")" + txt_oid="$(calc_oid "$(git cat-file -p :a.txt)")" + + git lfs migrate import --no-rewrite a.txt b/a.md + + # Ensure a.txt and subtree/a.md were imported, even though *.md only exists in the + # nested subtree/.gitattributes file + assert_pointer "refs/heads/master" "b/a.md" "$nested_md_oid" "140" + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "120" + + assert_local_object "$nested_md_oid" 140 + assert_local_object "$txt_oid" 120 + refute_local_object "$md_oid" 140 + + # Failure should occur when trying to import a.md as no entry exists in + # top-level .gitattributes file + git lfs migrate import --no-rewrite a.md 2>&1 | tee migrate.log + if [ ${PIPESTATUS[0]} -eq 0 ]; then + echo >&2 "fatal: expected git lfs migrate import --no-rewrite to fail, didn't" + exit 1 + fi + + grep "a.md did not match any Git LFS filters in .gitattributes" migrate.log +) +end_test + +begin_test "migrate import --no-rewrite (with commit message)" +( + set -e + + setup_local_branch_with_gitattrs + + prev_commit_oid="$(git rev-parse HEAD)" + expected_commit_msg="run git-lfs migrate import --no-rewrite" + + git lfs migrate import --message "$expected_commit_msg" --no-rewrite *.txt + + # Ensure the git history remained the same + new_commit_oid="$(git rev-parse HEAD~1)" + if [ "$prev_commit_oid" != "$new_commit_oid" ]; then + exit 1 + fi + + # Ensure a new commit was made + new_head_oid="$(git rev-parse HEAD)" + if [ "$prev_commit_oid" == "$new_oid" ]; then + exit 1 + fi + + # Ensure the provided commit message was used + commit_msg="$(git log -1 --pretty=format:%s)" + if [ "$commit_msg" != "$expected_commit_msg" ]; then + exit 1 + fi +) +end_test + +begin_test "migrate import --no-rewrite (with empty commit message)" +( + set -e + + setup_local_branch_with_gitattrs + + prev_commit_oid="$(git rev-parse HEAD)" + + git lfs migrate import -m "" --no-rewrite *.txt + + # Ensure the git history remained the same + new_commit_oid="$(git rev-parse HEAD~1)" + if [ "$prev_commit_oid" != "$new_commit_oid" ]; then + exit 1 + fi + + # Ensure a new commit was made + new_head_oid="$(git rev-parse HEAD)" + if [ "$prev_commit_oid" == "$new_oid" ]; then + exit 1 + fi + + # Ensure the provided commit message was used + commit_msg="$(git log -1 --pretty=format:%s)" + if [ "$commit_msg" != "" ]; then + exit 1 + fi +) +end_test diff --git a/test/test-migrate-import.sh b/test/test-migrate-import.sh index 1c1ff4ba..22517d0b 100755 --- a/test/test-migrate-import.sh +++ b/test/test-migrate-import.sh @@ -616,3 +616,94 @@ begin_test "migrate import (handle copies of files)" [ "$oid_root" = "$oid_root_after_migration" ] ) end_test + +begin_test "migrate import (--object-map)" +( + set -e + + setup_multiple_local_branches + + output_dir=$(mktemp -d) + + git log --all --pretty='format:%H' > "${output_dir}/old_sha.txt" + git lfs migrate import --everything --object-map "${output_dir}/object-map.txt" + git log --all --pretty='format:%H' > "${output_dir}/new_sha.txt" + paste -d',' "${output_dir}/old_sha.txt" "${output_dir}/new_sha.txt" > "${output_dir}/expected-map.txt" + + diff -u <(sort "${output_dir}/expected-map.txt") <(sort "${output_dir}/object-map.txt") +) +end_test + +begin_test "migrate import (--include with space)" +( + set -e + + setup_local_branch_with_space + + oid="$(calc_oid "$(git cat-file -p :"a file.txt")")" + + git lfs migrate import --include "a file.txt" + + assert_pointer "refs/heads/master" "a file.txt" "$oid" 50 + cat .gitattributes + if [ 1 -ne "$(grep -c "a\[\[:space:\]\]file.txt" .gitattributes)" ]; then + echo >&2 "fatal: expected \"a[[:space:]]file.txt\" to appear in .gitattributes" + echo >&2 "fatal: got" + sed -e 's/^/ /g' < .gitattributes >&2 + exit 1 + fi +) +end_test + +begin_test "migrate import (handle symbolic link)" +( + set -e + + setup_local_branch_with_symlink + + txt_oid="$(calc_oid "$(git cat-file -p :a.txt)")" + link_oid="$(calc_oid "$(git cat-file -p :link.txt)")" + + git lfs migrate import --include="*.txt" + + assert_pointer "refs/heads/master" "a.txt" "$txt_oid" "120" + + assert_local_object "$txt_oid" "120" + # "link.txt" is a symbolic link so it should be not in LFS + refute_local_object "$link_oid" "5" +) +end_test + +begin_test "migrate import (commit --allow-empty)" +( + set -e + + reponame="migrate---allow-empty" + git init "$reponame" + cd "$reponame" + + git commit --allow-empty -m "initial commit" + + original_head="$(git rev-parse HEAD)" + git lfs migrate import --everything + migrated_head="$(git rev-parse HEAD)" + + assert_ref_unmoved "HEAD" "$original_head" "$migrated_head" +) +end_test + +begin_test "migrate import (multiple remotes)" +( + set -e + + setup_multiple_remotes + + original_master="$(git rev-parse master)" + + git lfs migrate import + + migrated_master="$(git rev-parse master)" + + assert_ref_unmoved "master" "$original_master" "$migrated_master" +) +end_test diff --git a/test/test-status.sh b/test/test-status.sh index cfe437e6..3c6ab029 100755 --- a/test/test-status.sh +++ b/test/test-status.sh @@ -119,6 +119,34 @@ begin_test "status --json" ) end_test +begin_test "status in a sub-directory" +( + set -e + + reponame="status-sub-directory" + git init "$reponame" + cd "$reponame" + + git lfs track "*.dat" + printf "asdf" > file.dat + mkdir -p dir + git add .gitattributes file.dat + git commit -m "initial commit" + + printf "ASDF" > file.dat + + expected="On branch master + +Git LFS objects to be committed: + + +Git LFS objects not staged for commit: + + ../file.dat (LFS: f0e4c2f -> File: 99b3bcf)" + + [ "$expected" = "$(cd dir && git lfs status)" ] +) +end_test begin_test "status: outside git repository" ( diff --git a/test/test-track.sh b/test/test-track.sh index 65ec3423..3823a1c7 100755 --- a/test/test-track.sh +++ b/test/test-track.sh @@ -540,3 +540,58 @@ begin_test "track (with comments)" [ "0" -eq "$(grep -c "\.png" track.log)" ] ) end_test + +begin_test "track (with current-directory prefix)" +( + set -e + + reponame="track-with-current-directory-prefix" + git init "$reponame" + cd "$reponame" + + git lfs track "./a.dat" + printf "a" > a.dat + + git add .gitattributes a.dat + git commit -m "initial commit" + + grep -e "^a.dat" .gitattributes +) +end_test + +begin_test "track (global gitattributes)" +( + set -e + + reponame="track-global-gitattributes" + git init "$reponame" + cd "$reponame" + + global="$(cd .. && pwd)/gitattributes-global" + + echo "*.dat filter=lfs diff=lfs merge=lfs -text" > "$global" + git config --local core.attributesfile "$global" + + git lfs track 2>&1 | tee track.log + grep "*.dat" track.log +) +end_test + +begin_test "track (system gitattributes)" +( + set -e + + reponame="track-system-gitattributes" + git init "$reponame" + cd "$reponame" + + pushd "$TRASHDIR" > /dev/null + mkdir -p "prefix/${reponame}/etc" + cd "prefix/${reponame}/etc" + echo "*.dat filter=lfs diff=lfs merge=lfs -text" > gitattributes + popd > /dev/null + + PREFIX="${TRASHDIR}/prefix/${reponame}" git lfs track 2>&1 | tee track.log + grep "*.dat" track.log +) +end_test diff --git a/test/test-uninstall.sh b/test/test-uninstall.sh index 4d588bc0..82b07346 100755 --- a/test/test-uninstall.sh +++ b/test/test-uninstall.sh @@ -169,7 +169,12 @@ begin_test "uninstall --local" [ "global clean" = "$(git config --global filter.lfs.clean)" ] [ "global filter" = "$(git config --global filter.lfs.process)" ] - git lfs uninstall --local + git lfs uninstall --local 2>&1 | tee uninstall.log + if [ ${PIPESTATUS[0]} -ne 0 ]; then + echo >&2 "fatal: expected 'git lfs uninstall --local' to succeed" + exit 1 + fi + grep -v "Global Git LFS configuration has been removed." uninstall.log # global configs [ "global smudge" = "$(git config --global filter.lfs.smudge)" ] diff --git a/test/test-untrack.sh b/test/test-untrack.sh index e484c9fe..2e5005ea 100755 --- a/test/test-untrack.sh +++ b/test/test-untrack.sh @@ -72,3 +72,65 @@ begin_test "untrack removes escape sequences" assert_attributes_count "\\#" "filter=lfs" 0 ) end_test + +begin_test "untrack removes prefixed patterns (legacy)" +( + set -e + + reponame="untrack-removes-prefix-patterns-legacy" + git init "$reponame" + cd "$reponame" + + echo "./a.dat filter=lfs diff=lfs merge=lfs" > .gitattributes + printf "a" > a.dat + git add .gitattributes a.dat + git commit -m "initial commit" + + git lfs untrack "./a.dat" + + if [ ! -z "$(cat .gitattributes)" ]; then + echo &>2 "fatal: expected 'git lfs untrack' to clear .gitattributes" + exit 1 + fi + + git checkout -- .gitattributes + + git lfs untrack "a.dat" + + if [ ! -z "$(cat .gitattributes)" ]; then + echo &>2 "fatal: expected 'git lfs untrack' to clear .gitattributes" + exit 1 + fi +) +end_test + +begin_test "untrack removes prefixed patterns (modern)" +( + set -e + + reponame="untrack-removes-prefix-patterns-modern" + git init "$reponame" + cd "$reponame" + + echo "a.dat filter=lfs diff=lfs merge=lfs" > .gitattributes + printf "a" > a.dat + git add .gitattributes a.dat + git commit -m "initial commit" + + git lfs untrack "./a.dat" + + if [ ! -z "$(cat .gitattributes)" ]; then + echo &>2 "fatal: expected 'git lfs untrack' to clear .gitattributes" + exit 1 + fi + + git checkout -- .gitattributes + + git lfs untrack "a.dat" + + if [ ! -z "$(cat .gitattributes)" ]; then + echo &>2 "fatal: expected 'git lfs untrack' to clear .gitattributes" + exit 1 + fi +) +end_test diff --git a/test/test-version.sh b/test/test-version.sh new file mode 100755 index 00000000..31dfdb7f --- /dev/null +++ b/test/test-version.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +. "test/testlib.sh" + +begin_test "git lfs --version is a synonym of git lfs version" +( + set -e + + reponame="git-lfs-version-synonymous" + mkdir "$reponame" + cd "$reponame" + + git lfs version 2>&1 >version.log + git lfs --version 2>&1 >flag.log + + if [ "$(cat version.log)" != "$(cat flag.log)" ]; then + echo >&2 "fatal: expected 'git lfs version' and 'git lfs --version' to" + echo >&2 "produce identical output ..." + + diff -u {version,flag}.log + fi +) +end_test diff --git a/test/test-worktree.sh b/test/test-worktree.sh index 2b475361..d49f99a0 100755 --- a/test/test-worktree.sh +++ b/test/test-worktree.sh @@ -46,7 +46,7 @@ UploadTransfers=basic $(escape_path "$(env | grep "^GIT")") %s " "$(git lfs version)" "$(git version)" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" worktreename="worktree-2" @@ -82,7 +82,7 @@ UploadTransfers=basic $(escape_path "$(env | grep "^GIT")") %s " "$(git lfs version)" "$(git version)" "$envInitConfig") - actual=$(git lfs env) + actual=$(git lfs env | grep -v "^GIT_EXEC_PATH=") contains_same_elements "$expected" "$actual" ) end_test diff --git a/test/testhelpers.sh b/test/testhelpers.sh index bb8fdf81..40180933 100644 --- a/test/testhelpers.sh +++ b/test/testhelpers.sh @@ -738,3 +738,14 @@ has_test_dir() { exit 0 fi } + +add_symlink() { + local src=$1 + local dest=$2 + + prefix=`git rev-parse --show-prefix` + hashsrc=`printf "$src" | git hash-object -w --stdin` + + git update-index --add --cacheinfo 120000 "$hashsrc" "$prefix$dest" + git checkout -- "$dest" +} diff --git a/tq/meter.go b/tq/meter.go index a42a2cbb..36a6ae2d 100644 --- a/tq/meter.go +++ b/tq/meter.go @@ -5,7 +5,6 @@ import ( "math" "os" "path/filepath" - "strings" "sync" "sync/atomic" "time" @@ -231,12 +230,10 @@ func (m *Meter) skipUpdate() bool { func (m *Meter) str() string { // (Uploading|Downloading) LFS objects: 100% (10/10) 100 MiB | 10 MiB/s - - direction := strings.Title(m.Direction.String()) + "ing" percentage := 100 * float64(m.finishedFiles) / float64(m.estimatedFiles) return fmt.Sprintf("%s LFS objects: %3.f%% (%d/%d), %s | %s", - direction, + m.Direction.Verb(), percentage, m.finishedFiles, m.estimatedFiles, humanize.FormatBytes(clamp(m.currentBytes)), diff --git a/tq/transfer.go b/tq/transfer.go index b6a31ecc..60484e74 100644 --- a/tq/transfer.go +++ b/tq/transfer.go @@ -16,10 +16,27 @@ type Direction int const ( Upload = Direction(iota) Download = Direction(iota) + Checkout = Direction(iota) ) +// Verb returns a string containing the verb form of the receiving action. +func (d Direction) Verb() string { + switch d { + case Checkout: + return "Checking out" + case Download: + return "Downloading" + case Upload: + return "Uploading" + default: + return "" + } +} + func (d Direction) String() string { switch d { + case Checkout: + return "checkout" case Download: return "download" case Upload: diff --git a/vendor/github.com/bgentry/go-netrc/.hgignore b/vendor/github.com/git-lfs/go-netrc/.hgignore similarity index 100% rename from vendor/github.com/bgentry/go-netrc/.hgignore rename to vendor/github.com/git-lfs/go-netrc/.hgignore diff --git a/vendor/github.com/bgentry/go-netrc/LICENSE b/vendor/github.com/git-lfs/go-netrc/LICENSE similarity index 100% rename from vendor/github.com/bgentry/go-netrc/LICENSE rename to vendor/github.com/git-lfs/go-netrc/LICENSE diff --git a/vendor/github.com/bgentry/go-netrc/README.md b/vendor/github.com/git-lfs/go-netrc/README.md similarity index 100% rename from vendor/github.com/bgentry/go-netrc/README.md rename to vendor/github.com/git-lfs/go-netrc/README.md diff --git a/vendor/github.com/bgentry/go-netrc/netrc/examples/bad_default_order.netrc b/vendor/github.com/git-lfs/go-netrc/netrc/examples/bad_default_order.netrc similarity index 100% rename from vendor/github.com/bgentry/go-netrc/netrc/examples/bad_default_order.netrc rename to vendor/github.com/git-lfs/go-netrc/netrc/examples/bad_default_order.netrc diff --git a/vendor/github.com/bgentry/go-netrc/netrc/examples/good.netrc b/vendor/github.com/git-lfs/go-netrc/netrc/examples/good.netrc similarity index 79% rename from vendor/github.com/bgentry/go-netrc/netrc/examples/good.netrc rename to vendor/github.com/git-lfs/go-netrc/netrc/examples/good.netrc index 41a8e5ba..10e37083 100644 --- a/vendor/github.com/bgentry/go-netrc/netrc/examples/good.netrc +++ b/vendor/github.com/git-lfs/go-netrc/netrc/examples/good.netrc @@ -16,6 +16,12 @@ machine ray login demo password mypassword machine weirdlogin login uname password pass#pass +machine google.com + login alice@google.com + not-a-keyword + password secure + also-not-a-keyword + default login anonymous password joe@example.com diff --git a/vendor/github.com/bgentry/go-netrc/netrc/netrc.go b/vendor/github.com/git-lfs/go-netrc/netrc/netrc.go similarity index 98% rename from vendor/github.com/bgentry/go-netrc/netrc/netrc.go rename to vendor/github.com/git-lfs/go-netrc/netrc/netrc.go index ea49987c..4c9f8681 100644 --- a/vendor/github.com/bgentry/go-netrc/netrc/netrc.go +++ b/vendor/github.com/git-lfs/go-netrc/netrc/netrc.go @@ -24,6 +24,7 @@ const ( tkMacdef tkComment tkWhitespace + tkUnknown ) var keywords = map[string]tkType{ @@ -70,7 +71,7 @@ func (n *Netrc) MarshalText() (text []byte, err error) { // TODO(bgentry): not safe for concurrency for i := range n.tokens { switch n.tokens[i].kind { - case tkComment, tkDefault, tkWhitespace: // always append these types + case tkComment, tkDefault, tkWhitespace, tkUnknown: // always append these types text = append(text, n.tokens[i].rawkind...) default: if n.tokens[i].value != "" { // skip empty-value tokens @@ -391,9 +392,11 @@ func parse(r io.Reader, pos int) (*Netrc, error) { t, err = newToken(rawb) if err != nil { if currentMacro == nil { - return nil, &Error{pos, err.Error()} + t.kind = tkUnknown + nrc.tokens = append(nrc.tokens, t) + } else { + currentMacro.rawvalue = append(currentMacro.rawvalue, rawb...) } - currentMacro.rawvalue = append(currentMacro.rawvalue, rawb...) continue } diff --git a/vendor/github.com/bgentry/go-netrc/netrc/netrc_test.go b/vendor/github.com/git-lfs/go-netrc/netrc/netrc_test.go similarity index 99% rename from vendor/github.com/bgentry/go-netrc/netrc/netrc_test.go rename to vendor/github.com/git-lfs/go-netrc/netrc/netrc_test.go index 70ceacf6..d8437bf2 100644 --- a/vendor/github.com/bgentry/go-netrc/netrc/netrc_test.go +++ b/vendor/github.com/git-lfs/go-netrc/netrc/netrc_test.go @@ -18,6 +18,7 @@ var expectedMachines = []*Machine{ &Machine{Name: "mail.google.com", Login: "joe@gmail.com", Password: "somethingSecret", Account: "justagmail"}, &Machine{Name: "ray", Login: "demo", Password: "mypassword", Account: ""}, &Machine{Name: "weirdlogin", Login: "uname", Password: "pass#pass", Account: ""}, + &Machine{Name: "google.com", Login: "alice@google.com", Password: "secure"}, &Machine{Name: "", Login: "anonymous", Password: "joe@example.com", Account: ""}, } var expectedMacros = Macros{ @@ -146,7 +147,7 @@ func TestFindMachine(t *testing.T) { if err != nil { t.Fatal(err) } - if !eqMachine(m, expectedMachines[3]) { + if !eqMachine(m, expectedMachines[4]) { t.Errorf("bad machine; expected %v, got %v\n", expectedMachines[3], m) } if !m.IsDefault() {