wip

.travis.yml

@@ -11,7 +11,7 @@ env:
   global:
     - GIT_LFS_TEST_DIR="$HOME/git-lfs-tests"
     - GIT_SOURCE_REPO="https://github.com/git/git.git"
-    - GIT_SOURCE_BRANCH="master"
+    - GIT_SOURCE_BRANCH="next"
 
 matrix:
   fast_finish: true
@@ -26,6 +26,20 @@ matrix:
           make --jobs=2;
           make install;
           cd ..;
+    - env: git-from-source
+      os: osx
+      before_script:
+        - >
+          export NO_OPENSSL=YesPlease;
+          export APPLE_COMMON_CRYPTO=YesPlease;
+          brew install gettext;
+          brew link --force gettext;
+          git clone $GIT_SOURCE_REPO git-source;
+          cd git-source;
+          git checkout $GIT_SOURCE_BRANCH;
+          make --jobs=2;
+          make install;
+          cd ..;
     - env: git-latest
       os: linux
       addons:

commands/command_env.go

@@ -39,7 +39,7 @@ func envCommand(cmd *cobra.Command, args []string) {
 		Print(env)
 	}
 
-	for _, key := range []string{"filter.lfs.smudge", "filter.lfs.clean"} {
+	for _, key := range []string{"filter.lfs.process", "filter.lfs.smudge", "filter.lfs.clean"} {
 		value, _ := cfg.Git.Get(key)
 		Print("git config %s = %q", key, value)
 	}

commands/command_filter.go

@@ -0,0 +1,165 @@
+package commands
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/github/git-lfs/config"
+	"github.com/github/git-lfs/errors"
+	"github.com/github/git-lfs/git"
+	"github.com/github/git-lfs/lfs"
+	"github.com/github/git-lfs/progress"
+	"github.com/spf13/cobra"
+)
+
+var (
+	filterSmudgeSkip = false
+)
+
+func clean(reader io.Reader, fileName string) ([]byte, error) {
+	var cb progress.CopyCallback
+	var file *os.File
+	var fileSize int64
+	if len(fileName) > 0 {
+		stat, err := os.Stat(fileName)
+		if err == nil && stat != nil {
+			fileSize = stat.Size()
+
+			localCb, localFile, err := lfs.CopyCallbackFile("clean", fileName, 1, 1)
+			if err != nil {
+				Error(err.Error())
+			} else {
+				cb = localCb
+				file = localFile
+			}
+		}
+	}
+
+	cleaned, err := lfs.PointerClean(reader, fileName, fileSize, cb)
+	if file != nil {
+		file.Close()
+	}
+
+	if cleaned != nil {
+		defer cleaned.Teardown()
+	}
+
+	if errors.IsCleanPointerError(err) {
+		// TODO: report errors differently!
+		// os.Stdout.Write(errors.GetContext(err, "bytes").([]byte))
+		return errors.GetContext(err, "bytes").([]byte), nil
+	}
+
+	if err != nil {
+		Panic(err, "Error cleaning asset.")
+	}
+
+	tmpfile := cleaned.Filename
+	mediafile, err := lfs.LocalMediaPath(cleaned.Oid)
+	if err != nil {
+		Panic(err, "Unable to get local media path.")
+	}
+
+	if stat, _ := os.Stat(mediafile); stat != nil {
+		if stat.Size() != cleaned.Size && len(cleaned.Pointer.Extensions) == 0 {
+			Exit("Files don't match:\n%s\n%s", mediafile, tmpfile)
+		}
+		Debug("%s exists", mediafile)
+	} else {
+		if err := os.Rename(tmpfile, mediafile); err != nil {
+			Panic(err, "Unable to move %s to %s\n", tmpfile, mediafile)
+		}
+
+		Debug("Writing %s", mediafile)
+	}
+
+	return []byte(cleaned.Pointer.Encoded()), nil
+}
+
+func smudge(reader io.Reader, filename string) ([]byte, error) {
+	ptr, err := lfs.DecodePointer(reader)
+	if err != nil {
+		// mr := io.MultiReader(b, reader)
+		// _, err := io.Copy(os.Stdout, mr)
+		// if err != nil {
+		// 	Panic(err, "Error writing data to stdout:")
+		// }
+		var content []byte
+		reader.Read(content)
+		return content, nil
+	}
+
+	lfs.LinkOrCopyFromReference(ptr.Oid, ptr.Size)
+
+	cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1)
+	if err != nil {
+		Error(err.Error())
+	}
+
+	cfg := config.Config
+	download := lfs.FilenamePassesIncludeExcludeFilter(filename, cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
+
+	if filterSmudgeSkip || cfg.Os.Bool("GIT_LFS_SKIP_SMUDGE", false) {
+		download = false
+	}
+
+	buf := new(bytes.Buffer)
+	err = ptr.Smudge(buf, filename, download, TransferManifest(), cb)
+	if file != nil {
+		file.Close()
+	}
+
+	if err != nil {
+		// Download declined error is ok to skip if we weren't requesting download
+		if !(errors.IsDownloadDeclinedError(err) && !download) {
+			LoggedError(err, "Error downloading object: %s (%s)", filename, ptr.Oid)
+			if !cfg.SkipDownloadErrors() {
+				// TODO: What to do best here?
+				os.Exit(2)
+			}
+		}
+
+		return []byte(ptr.Encoded()), nil
+	}
+
+	return buf.Bytes(), nil
+}
+
+func filterCommand(cmd *cobra.Command, args []string) {
+	requireStdin("This command should be run by the Git filter process")
+	lfs.InstallHooks(false)
+
+	s := git.NewObjectScanner(os.Stdin, os.Stdout)
+	s.Init()
+	s.NegotiateCapabilities()
+	for {
+		request, data, err := s.ReadRequest()
+		if err != nil {
+			break
+		}
+
+		// TODO:
+		// ReadRequest should return data as Reader instead of []byte ?!
+		// clean/smudge should also take a Writer instead of returning []byte
+		var outputData []byte
+		switch request["command"] {
+		case "clean":
+			outputData, _ = clean(bytes.NewReader(data), request["pathname"])
+		case "smudge":
+			outputData, _ = smudge(bytes.NewReader(data), request["pathname"])
+		default:
+			fmt.Errorf("Unknown command %s", cmd)
+			break
+		}
+
+		s.WriteResponse(outputData)
+	}
+}
+
+func init() {
+	RegisterCommand("filter", filterCommand, func(cmd *cobra.Command) {
+		cmd.Flags().BoolVarP(&filterSmudgeSkip, "skip", "s", false, "")
+	})
+}

git/git.go

@@ -809,6 +809,7 @@ func CloneWithoutFilters(flags CloneFlags, args []string) error {
 	// with --skip-smudge is costly across many files in a checkout
 	cmdargs := []string{
 		"-c", fmt.Sprintf("filter.lfs.smudge=%v", filterOverride),
+		"-c", "filter.lfs.process=",
 		"-c", "filter.lfs.required=false",
 		"clone"}
 

git/git_filter_protocol.go

@@ -0,0 +1,248 @@
+// Package git contains various commands that shell out to git
+// NOTE: Subject to change, do not rely on this package from outside git-lfs source
+package git
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/github/git-lfs/errors"
+	"github.com/rubyist/tracerx"
+)
+
+const (
+	MaxPacketLenght = 65516
+)
+
+// Private function copied from "github.com/xeipuuv/gojsonschema/utils.go"
+// TODO: Is there a way to reuse this?
+func isStringInSlice(s []string, what string) bool {
+	for i := range s {
+		if s[i] == what {
+			return true
+		}
+	}
+	return false
+}
+
+type ObjectScanner struct {
+	r *bufio.Reader
+	w *bufio.Writer
+}
+
+func NewObjectScanner(r io.Reader, w io.Writer) *ObjectScanner {
+	return &ObjectScanner{
+		r: bufio.NewReader(r),
+		w: bufio.NewWriter(w),
+	}
+}
+
+func (o *ObjectScanner) readPacket() ([]byte, error) {
+	pktLenHex, err := ioutil.ReadAll(io.LimitReader(o.r, 4))
+	if err != nil || len(pktLenHex) != 4 { // TODO check pktLenHex length
+		return nil, err
+	}
+	pktLen, err := strconv.ParseInt(string(pktLenHex), 16, 0)
+	if err != nil {
+		return nil, err
+	}
+	if pktLen == 0 {
+		return nil, nil
+	} else if pktLen <= 4 {
+		return nil, errors.New("Invalid packet length.")
+	}
+	return ioutil.ReadAll(io.LimitReader(o.r, pktLen-4))
+}
+
+func (o *ObjectScanner) readPacketText() (string, error) {
+	data, err := o.readPacket()
+	return strings.TrimSuffix(string(data), "\n"), err
+}
+
+func (o *ObjectScanner) readPacketList() ([]string, error) {
+	var list []string
+	for {
+		data, err := o.readPacketText()
+		if err != nil {
+			return nil, err
+		}
+		if len(data) == 0 {
+			break
+		}
+		list = append(list, data)
+	}
+	return list, nil
+}
+
+func (o *ObjectScanner) writePacket(data []byte) error {
+	if len(data) > MaxPacketLenght {
+		return errors.New("Packet length exceeds maximal length")
+	}
+	_, err := o.w.WriteString(fmt.Sprintf("%04x", len(data)+4))
+	if err != nil {
+		return err
+	}
+	_, err = o.w.Write(data)
+	if err != nil {
+		return err
+	}
+	err = o.w.Flush()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (o *ObjectScanner) writeFlush() error {
+	_, err := o.w.WriteString(fmt.Sprintf("%04x", 0))
+	if err != nil {
+		return err
+	}
+	err = o.w.Flush()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (o *ObjectScanner) writePacketText(data string) error {
+	//TODO: there is probably a more efficient way to do this. worth it?
+	return o.writePacket([]byte(data + "\n"))
+}
+
+func (o *ObjectScanner) writePacketList(list []string) error {
+	for _, i := range list {
+		err := o.writePacketText(i)
+		if err != nil {
+			return err
+		}
+	}
+	return o.writeFlush()
+}
+
+func (o *ObjectScanner) writeStatus(status string) error {
+	return o.writePacketList([]string{"status=" + status})
+}
+
+func (o *ObjectScanner) Init() bool {
+	tracerx.Printf("Initialize filter")
+	reqVer := "version=2"
+
+	initMsg, err := o.readPacketText()
+	if err != nil {
+		fmt.Fprintf(os.Stderr,
+			"Error: reading filter initialization failed with %s\n", err)
+		return false
+	}
+	if initMsg != "git-filter-client" {
+		fmt.Fprintf(os.Stderr,
+			"Error: invalid filter protocol welcome message: %s\n", initMsg)
+		return false
+	}
+
+	supVers, err := o.readPacketList()
+	if err != nil {
+		fmt.Fprintf(os.Stderr,
+			"Error: reading filter versions failed with %s\n", err)
+		return false
+	}
+	if !isStringInSlice(supVers, reqVer) {
+		fmt.Fprintf(os.Stderr,
+			"Error: filter '%s' not supported (your Git supports: %s)\n",
+			reqVer, supVers)
+		return false
+	}
+
+	err = o.writePacketList([]string{"git-filter-server", reqVer})
+	if err != nil {
+		fmt.Fprintf(os.Stderr,
+			"Error: writing filter initialization failed with %s\n", err)
+		return false
+	}
+	return true
+}
+
+func (o *ObjectScanner) NegotiateCapabilities() bool {
+	reqCaps := []string{"capability=clean", "capability=smudge"}
+
+	supCaps, err := o.readPacketList()
+	if err != nil {
+		fmt.Fprintf(os.Stderr,
+			"Error: reading filter capabilities failed with %s\n", err)
+		return false
+	}
+	for _, reqCap := range reqCaps {
+		if !isStringInSlice(supCaps, reqCap) {
+			fmt.Fprintf(os.Stderr,
+				"Error: filter '%s' not supported (your Git supports: %s)\n",
+				reqCap, supCaps)
+			return false
+		}
+	}
+
+	err = o.writePacketList(reqCaps)
+	if err != nil {
+		fmt.Fprintf(os.Stderr,
+			"Error: writing filter capabilities failed with %s\n", err)
+		return false
+	}
+
+	return true
+}
+
+func (o *ObjectScanner) ReadRequest() (map[string]string, []byte, error) {
+	tracerx.Printf("Process filter command.")
+
+	requestList, err := o.readPacketList()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	requestMap := make(map[string]string)
+	for _, pair := range requestList {
+		v := strings.Split(pair, "=")
+		requestMap[v[0]] = v[1]
+	}
+
+	var data []byte
+	for {
+		chunk, err := o.readPacket()
+		if err != nil {
+			// TODO: should we check the err of this call, to?!
+			o.writeStatus("error")
+			return nil, nil, err
+		}
+		if len(chunk) == 0 {
+			break
+		}
+		data = append(data, chunk...) // probably more efficient way?!
+	}
+	o.writeStatus("success")
+	return requestMap, data, nil
+}
+
+func (o *ObjectScanner) WriteResponse(outputData []byte) error {
+	for {
+		chunkSize := len(outputData)
+		if chunkSize == 0 {
+			o.writeFlush()
+			break
+		} else if chunkSize > MaxPacketLenght {
+			chunkSize = MaxPacketLenght // TODO check packets with the exact size
+		}
+		err := o.writePacket(outputData[:chunkSize])
+		if err != nil {
+			// TODO: should we check the err of this call, to?!
+			o.writeStatus("error")
+			return err
+		}
+		outputData = outputData[chunkSize:]
+	}
+	o.writeStatus("success")
+	return nil
+}

lfs/setup.go

@@ -28,11 +28,13 @@ var (
 		Properties: map[string]string{
 			"clean":    "git-lfs clean -- %f",
 			"smudge":   "git-lfs smudge -- %f",
+			"process":  "git-lfs filter",
 			"required": "true",
 		},
 		Upgradeables: map[string][]string{
 			"clean":  []string{"git-lfs clean %f"},
 			"smudge": []string{"git-lfs smudge %f"},
+			// TODO: process here, too?
 		},
 	}
 
@@ -41,6 +43,7 @@ var (
 		Properties: map[string]string{
 			"clean":    "git-lfs clean -- %f",
 			"smudge":   "git-lfs smudge --skip -- %f",
+			"process":  "git-lfs filter --skip",
 			"required": "true",
 		},
 		Upgradeables: map[string][]string{

script/integration

@@ -41,4 +41,4 @@ fi
 
 setup
 
-GO15VENDOREXPERIMENT=1 GIT_LFS_TEST_MAXPROCS=$GIT_LFS_TEST_MAXPROCS GIT_LFS_TEST_DIR="$GIT_LFS_TEST_DIR" SHUTDOWN_LFS="no" go run script/*.go -cmd integration "$@"
+GO15VENDOREXPERIMENT=1 GIT_LFS_USE_LEGACY_FILTER=$GIT_LFS_USE_LEGACY_FILTER GIT_LFS_TEST_MAXPROCS=$GIT_LFS_TEST_MAXPROCS GIT_LFS_TEST_DIR="$GIT_LFS_TEST_DIR" SHUTDOWN_LFS="no" go run script/*.go -cmd integration "$@"

test/README.md

@@ -76,6 +76,7 @@ and the remote repository data in `test/remote`.
 * `SKIPCOMPILE=1` - This skips the Git LFS compilation step.  Speeds up the
 tests when you're running the same test script multiple times without changing
 any Go code.
+* `GIT_LFS_USE_LEGACY_FILTER=1` - TODO
 
 Also ensure that your `noproxy` environment variable contains `127.0.0.1` host,
 to allow git commands to reach the local Git server `lfstest-gitserver`.

test/test-batch-retries.sh

@@ -46,6 +46,7 @@ begin_test "batch storage download causes retries"
 
   pushd ..
     git \
+      -c "filter.lfs.process=" \
       -c "filter.lfs.smudge=cat" \
       -c "filter.lfs.required=false" \
       clone "$GITSERVER/$reponame" "$reponame-assert"

test/test-env.sh

@@ -2,7 +2,8 @@
 
 . "test/testlib.sh"
 
-envInitConfig='git config filter.lfs.smudge = "git-lfs smudge -- %f"
+envInitConfig='git config filter.lfs.process = "git-lfs filter"
+git config filter.lfs.smudge = "git-lfs smudge -- %f"
 git config filter.lfs.clean = "git-lfs clean -- %f"'
 
 begin_test "env with no remote"
@@ -616,8 +617,6 @@ AccessUpload=none
 DownloadTransfers=basic
 UploadTransfers=basic
 %s
-git config filter.lfs.smudge = \"\"
-git config filter.lfs.clean = \"\"
 ' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$envVars")
   actual5=$(GIT_DIR=$gitDir GIT_WORK_TREE=a/b git lfs env)
   contains_same_elements "$expected5" "$actual5"

test/test-install.sh

@@ -6,6 +6,7 @@ begin_test "install again"
 (
   set -e
 
+  # TODO: add process filter here
   smudge="$(git config filter.lfs.smudge)"
   clean="$(git config filter.lfs.clean)"
 

test/test-legacy-retries.sh

@@ -47,6 +47,7 @@ begin_test "legacy download check causes retries"
 
   pushd ..
     git \
+      -c "filter.lfs.process=" \
       -c "filter.lfs.smudge=cat" \
       -c "filter.lfs.required=false" \
       clone "$GITSERVER/$reponame" "$reponame-assert"
@@ -108,6 +109,7 @@ begin_test "legacy storage download causes retries"
 
   pushd ..
     git \
+      -c "filter.lfs.process=" \
       -c "filter.lfs.smudge=cat" \
       -c "filter.lfs.required=false" \
       clone "$GITSERVER/$reponame" "$reponame-assert"

test/test-uninstall.sh

@@ -6,6 +6,7 @@ begin_test "uninstall outside repository"
 (
   set -e
 
+  # TODO: add process filter here
   smudge="$(git config filter.lfs.smudge)"
   clean="$(git config filter.lfs.clean)"
 

test/test-worktree.sh

@@ -3,7 +3,8 @@
 . "test/testlib.sh"
 
 ensure_git_version_isnt $VERSION_LOWER "2.5.0"
-envInitConfig='git config filter.lfs.smudge = "git-lfs smudge -- %f"
+envInitConfig='git config filter.lfs.process = "git-lfs filter"
+git config filter.lfs.smudge = "git-lfs smudge -- %f"
 git config filter.lfs.clean = "git-lfs clean -- %f"'
 
 begin_test "git worktree"

test/testhelpers.sh

@@ -175,7 +175,7 @@ wait_for_file() {
   return 1
 }
 
-# setup_remote_repo intializes a bare Git repository that is accessible through
+# setup_remote_repo initializes a bare Git repository that is accessible through
 # the test Git server. The `pwd` is set to the repository's directory, in case
 # further commands need to be run. This server is running for every test in a
 # script/integration run, so every test file should setup its own remote
@@ -314,7 +314,12 @@ setup() {
   git config --global user.email "git-lfs@example.com"
   git config --global http.sslcainfo "$LFS_CERT_FILE"
 
-  grep "git-lfs clean" "$REMOTEDIR/home/.gitconfig" > /dev/null || {
+  if [ "$GIT_LFS_USE_LEGACY_FILTER" == "1" ]; then
+    FILTER="clean"
+  else
+    FILTER="filter"
+  fi
+  grep "git-lfs $FILTER" "$REMOTEDIR/home/.gitconfig" > /dev/null || {
     echo "global git config should be set in $REMOTEDIR/home"
     ls -al "$REMOTEDIR/home"
     exit 1