wip
This commit is contained in:
parent
a244865bb6
commit
d874af9ac1
16
.travis.yml
16
.travis.yml
@ -11,7 +11,7 @@ env:
|
||||
global:
|
||||
- GIT_LFS_TEST_DIR="$HOME/git-lfs-tests"
|
||||
- GIT_SOURCE_REPO="https://github.com/git/git.git"
|
||||
- GIT_SOURCE_BRANCH="master"
|
||||
- GIT_SOURCE_BRANCH="next"
|
||||
|
||||
matrix:
|
||||
fast_finish: true
|
||||
@ -26,6 +26,20 @@ matrix:
|
||||
make --jobs=2;
|
||||
make install;
|
||||
cd ..;
|
||||
- env: git-from-source
|
||||
os: osx
|
||||
before_script:
|
||||
- >
|
||||
export NO_OPENSSL=YesPlease;
|
||||
export APPLE_COMMON_CRYPTO=YesPlease;
|
||||
brew install gettext;
|
||||
brew link --force gettext;
|
||||
git clone $GIT_SOURCE_REPO git-source;
|
||||
cd git-source;
|
||||
git checkout $GIT_SOURCE_BRANCH;
|
||||
make --jobs=2;
|
||||
make install;
|
||||
cd ..;
|
||||
- env: git-latest
|
||||
os: linux
|
||||
addons:
|
||||
|
@ -39,7 +39,7 @@ func envCommand(cmd *cobra.Command, args []string) {
|
||||
Print(env)
|
||||
}
|
||||
|
||||
for _, key := range []string{"filter.lfs.smudge", "filter.lfs.clean"} {
|
||||
for _, key := range []string{"filter.lfs.process", "filter.lfs.smudge", "filter.lfs.clean"} {
|
||||
value, _ := cfg.Git.Get(key)
|
||||
Print("git config %s = %q", key, value)
|
||||
}
|
||||
|
165
commands/command_filter.go
Normal file
165
commands/command_filter.go
Normal file
@ -0,0 +1,165 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/github/git-lfs/config"
|
||||
"github.com/github/git-lfs/errors"
|
||||
"github.com/github/git-lfs/git"
|
||||
"github.com/github/git-lfs/lfs"
|
||||
"github.com/github/git-lfs/progress"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
filterSmudgeSkip = false
|
||||
)
|
||||
|
||||
func clean(reader io.Reader, fileName string) ([]byte, error) {
|
||||
var cb progress.CopyCallback
|
||||
var file *os.File
|
||||
var fileSize int64
|
||||
if len(fileName) > 0 {
|
||||
stat, err := os.Stat(fileName)
|
||||
if err == nil && stat != nil {
|
||||
fileSize = stat.Size()
|
||||
|
||||
localCb, localFile, err := lfs.CopyCallbackFile("clean", fileName, 1, 1)
|
||||
if err != nil {
|
||||
Error(err.Error())
|
||||
} else {
|
||||
cb = localCb
|
||||
file = localFile
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cleaned, err := lfs.PointerClean(reader, fileName, fileSize, cb)
|
||||
if file != nil {
|
||||
file.Close()
|
||||
}
|
||||
|
||||
if cleaned != nil {
|
||||
defer cleaned.Teardown()
|
||||
}
|
||||
|
||||
if errors.IsCleanPointerError(err) {
|
||||
// TODO: report errors differently!
|
||||
// os.Stdout.Write(errors.GetContext(err, "bytes").([]byte))
|
||||
return errors.GetContext(err, "bytes").([]byte), nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
Panic(err, "Error cleaning asset.")
|
||||
}
|
||||
|
||||
tmpfile := cleaned.Filename
|
||||
mediafile, err := lfs.LocalMediaPath(cleaned.Oid)
|
||||
if err != nil {
|
||||
Panic(err, "Unable to get local media path.")
|
||||
}
|
||||
|
||||
if stat, _ := os.Stat(mediafile); stat != nil {
|
||||
if stat.Size() != cleaned.Size && len(cleaned.Pointer.Extensions) == 0 {
|
||||
Exit("Files don't match:\n%s\n%s", mediafile, tmpfile)
|
||||
}
|
||||
Debug("%s exists", mediafile)
|
||||
} else {
|
||||
if err := os.Rename(tmpfile, mediafile); err != nil {
|
||||
Panic(err, "Unable to move %s to %s\n", tmpfile, mediafile)
|
||||
}
|
||||
|
||||
Debug("Writing %s", mediafile)
|
||||
}
|
||||
|
||||
return []byte(cleaned.Pointer.Encoded()), nil
|
||||
}
|
||||
|
||||
func smudge(reader io.Reader, filename string) ([]byte, error) {
|
||||
ptr, err := lfs.DecodePointer(reader)
|
||||
if err != nil {
|
||||
// mr := io.MultiReader(b, reader)
|
||||
// _, err := io.Copy(os.Stdout, mr)
|
||||
// if err != nil {
|
||||
// Panic(err, "Error writing data to stdout:")
|
||||
// }
|
||||
var content []byte
|
||||
reader.Read(content)
|
||||
return content, nil
|
||||
}
|
||||
|
||||
lfs.LinkOrCopyFromReference(ptr.Oid, ptr.Size)
|
||||
|
||||
cb, file, err := lfs.CopyCallbackFile("smudge", filename, 1, 1)
|
||||
if err != nil {
|
||||
Error(err.Error())
|
||||
}
|
||||
|
||||
cfg := config.Config
|
||||
download := lfs.FilenamePassesIncludeExcludeFilter(filename, cfg.FetchIncludePaths(), cfg.FetchExcludePaths())
|
||||
|
||||
if filterSmudgeSkip || cfg.Os.Bool("GIT_LFS_SKIP_SMUDGE", false) {
|
||||
download = false
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
err = ptr.Smudge(buf, filename, download, TransferManifest(), cb)
|
||||
if file != nil {
|
||||
file.Close()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// Download declined error is ok to skip if we weren't requesting download
|
||||
if !(errors.IsDownloadDeclinedError(err) && !download) {
|
||||
LoggedError(err, "Error downloading object: %s (%s)", filename, ptr.Oid)
|
||||
if !cfg.SkipDownloadErrors() {
|
||||
// TODO: What to do best here?
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
return []byte(ptr.Encoded()), nil
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func filterCommand(cmd *cobra.Command, args []string) {
|
||||
requireStdin("This command should be run by the Git filter process")
|
||||
lfs.InstallHooks(false)
|
||||
|
||||
s := git.NewObjectScanner(os.Stdin, os.Stdout)
|
||||
s.Init()
|
||||
s.NegotiateCapabilities()
|
||||
for {
|
||||
request, data, err := s.ReadRequest()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
// TODO:
|
||||
// ReadRequest should return data as Reader instead of []byte ?!
|
||||
// clean/smudge should also take a Writer instead of returning []byte
|
||||
var outputData []byte
|
||||
switch request["command"] {
|
||||
case "clean":
|
||||
outputData, _ = clean(bytes.NewReader(data), request["pathname"])
|
||||
case "smudge":
|
||||
outputData, _ = smudge(bytes.NewReader(data), request["pathname"])
|
||||
default:
|
||||
fmt.Errorf("Unknown command %s", cmd)
|
||||
break
|
||||
}
|
||||
|
||||
s.WriteResponse(outputData)
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterCommand("filter", filterCommand, func(cmd *cobra.Command) {
|
||||
cmd.Flags().BoolVarP(&filterSmudgeSkip, "skip", "s", false, "")
|
||||
})
|
||||
}
|
@ -809,6 +809,7 @@ func CloneWithoutFilters(flags CloneFlags, args []string) error {
|
||||
// with --skip-smudge is costly across many files in a checkout
|
||||
cmdargs := []string{
|
||||
"-c", fmt.Sprintf("filter.lfs.smudge=%v", filterOverride),
|
||||
"-c", "filter.lfs.process=",
|
||||
"-c", "filter.lfs.required=false",
|
||||
"clone"}
|
||||
|
||||
|
248
git/git_filter_protocol.go
Normal file
248
git/git_filter_protocol.go
Normal file
@ -0,0 +1,248 @@
|
||||
// Package git contains various commands that shell out to git
|
||||
// NOTE: Subject to change, do not rely on this package from outside git-lfs source
|
||||
package git
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/github/git-lfs/errors"
|
||||
"github.com/rubyist/tracerx"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxPacketLenght = 65516
|
||||
)
|
||||
|
||||
// Private function copied from "github.com/xeipuuv/gojsonschema/utils.go"
|
||||
// TODO: Is there a way to reuse this?
|
||||
func isStringInSlice(s []string, what string) bool {
|
||||
for i := range s {
|
||||
if s[i] == what {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type ObjectScanner struct {
|
||||
r *bufio.Reader
|
||||
w *bufio.Writer
|
||||
}
|
||||
|
||||
func NewObjectScanner(r io.Reader, w io.Writer) *ObjectScanner {
|
||||
return &ObjectScanner{
|
||||
r: bufio.NewReader(r),
|
||||
w: bufio.NewWriter(w),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) readPacket() ([]byte, error) {
|
||||
pktLenHex, err := ioutil.ReadAll(io.LimitReader(o.r, 4))
|
||||
if err != nil || len(pktLenHex) != 4 { // TODO check pktLenHex length
|
||||
return nil, err
|
||||
}
|
||||
pktLen, err := strconv.ParseInt(string(pktLenHex), 16, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if pktLen == 0 {
|
||||
return nil, nil
|
||||
} else if pktLen <= 4 {
|
||||
return nil, errors.New("Invalid packet length.")
|
||||
}
|
||||
return ioutil.ReadAll(io.LimitReader(o.r, pktLen-4))
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) readPacketText() (string, error) {
|
||||
data, err := o.readPacket()
|
||||
return strings.TrimSuffix(string(data), "\n"), err
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) readPacketList() ([]string, error) {
|
||||
var list []string
|
||||
for {
|
||||
data, err := o.readPacketText()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
break
|
||||
}
|
||||
list = append(list, data)
|
||||
}
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) writePacket(data []byte) error {
|
||||
if len(data) > MaxPacketLenght {
|
||||
return errors.New("Packet length exceeds maximal length")
|
||||
}
|
||||
_, err := o.w.WriteString(fmt.Sprintf("%04x", len(data)+4))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = o.w.Write(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = o.w.Flush()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) writeFlush() error {
|
||||
_, err := o.w.WriteString(fmt.Sprintf("%04x", 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = o.w.Flush()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) writePacketText(data string) error {
|
||||
//TODO: there is probably a more efficient way to do this. worth it?
|
||||
return o.writePacket([]byte(data + "\n"))
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) writePacketList(list []string) error {
|
||||
for _, i := range list {
|
||||
err := o.writePacketText(i)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return o.writeFlush()
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) writeStatus(status string) error {
|
||||
return o.writePacketList([]string{"status=" + status})
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) Init() bool {
|
||||
tracerx.Printf("Initialize filter")
|
||||
reqVer := "version=2"
|
||||
|
||||
initMsg, err := o.readPacketText()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"Error: reading filter initialization failed with %s\n", err)
|
||||
return false
|
||||
}
|
||||
if initMsg != "git-filter-client" {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"Error: invalid filter protocol welcome message: %s\n", initMsg)
|
||||
return false
|
||||
}
|
||||
|
||||
supVers, err := o.readPacketList()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"Error: reading filter versions failed with %s\n", err)
|
||||
return false
|
||||
}
|
||||
if !isStringInSlice(supVers, reqVer) {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"Error: filter '%s' not supported (your Git supports: %s)\n",
|
||||
reqVer, supVers)
|
||||
return false
|
||||
}
|
||||
|
||||
err = o.writePacketList([]string{"git-filter-server", reqVer})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"Error: writing filter initialization failed with %s\n", err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) NegotiateCapabilities() bool {
|
||||
reqCaps := []string{"capability=clean", "capability=smudge"}
|
||||
|
||||
supCaps, err := o.readPacketList()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"Error: reading filter capabilities failed with %s\n", err)
|
||||
return false
|
||||
}
|
||||
for _, reqCap := range reqCaps {
|
||||
if !isStringInSlice(supCaps, reqCap) {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"Error: filter '%s' not supported (your Git supports: %s)\n",
|
||||
reqCap, supCaps)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
err = o.writePacketList(reqCaps)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"Error: writing filter capabilities failed with %s\n", err)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) ReadRequest() (map[string]string, []byte, error) {
|
||||
tracerx.Printf("Process filter command.")
|
||||
|
||||
requestList, err := o.readPacketList()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
requestMap := make(map[string]string)
|
||||
for _, pair := range requestList {
|
||||
v := strings.Split(pair, "=")
|
||||
requestMap[v[0]] = v[1]
|
||||
}
|
||||
|
||||
var data []byte
|
||||
for {
|
||||
chunk, err := o.readPacket()
|
||||
if err != nil {
|
||||
// TODO: should we check the err of this call, to?!
|
||||
o.writeStatus("error")
|
||||
return nil, nil, err
|
||||
}
|
||||
if len(chunk) == 0 {
|
||||
break
|
||||
}
|
||||
data = append(data, chunk...) // probably more efficient way?!
|
||||
}
|
||||
o.writeStatus("success")
|
||||
return requestMap, data, nil
|
||||
}
|
||||
|
||||
func (o *ObjectScanner) WriteResponse(outputData []byte) error {
|
||||
for {
|
||||
chunkSize := len(outputData)
|
||||
if chunkSize == 0 {
|
||||
o.writeFlush()
|
||||
break
|
||||
} else if chunkSize > MaxPacketLenght {
|
||||
chunkSize = MaxPacketLenght // TODO check packets with the exact size
|
||||
}
|
||||
err := o.writePacket(outputData[:chunkSize])
|
||||
if err != nil {
|
||||
// TODO: should we check the err of this call, to?!
|
||||
o.writeStatus("error")
|
||||
return err
|
||||
}
|
||||
outputData = outputData[chunkSize:]
|
||||
}
|
||||
o.writeStatus("success")
|
||||
return nil
|
||||
}
|
@ -28,11 +28,13 @@ var (
|
||||
Properties: map[string]string{
|
||||
"clean": "git-lfs clean -- %f",
|
||||
"smudge": "git-lfs smudge -- %f",
|
||||
"process": "git-lfs filter",
|
||||
"required": "true",
|
||||
},
|
||||
Upgradeables: map[string][]string{
|
||||
"clean": []string{"git-lfs clean %f"},
|
||||
"smudge": []string{"git-lfs smudge %f"},
|
||||
// TODO: process here, too?
|
||||
},
|
||||
}
|
||||
|
||||
@ -41,6 +43,7 @@ var (
|
||||
Properties: map[string]string{
|
||||
"clean": "git-lfs clean -- %f",
|
||||
"smudge": "git-lfs smudge --skip -- %f",
|
||||
"process": "git-lfs filter --skip",
|
||||
"required": "true",
|
||||
},
|
||||
Upgradeables: map[string][]string{
|
||||
|
@ -41,4 +41,4 @@ fi
|
||||
|
||||
setup
|
||||
|
||||
GO15VENDOREXPERIMENT=1 GIT_LFS_TEST_MAXPROCS=$GIT_LFS_TEST_MAXPROCS GIT_LFS_TEST_DIR="$GIT_LFS_TEST_DIR" SHUTDOWN_LFS="no" go run script/*.go -cmd integration "$@"
|
||||
GO15VENDOREXPERIMENT=1 GIT_LFS_USE_LEGACY_FILTER=$GIT_LFS_USE_LEGACY_FILTER GIT_LFS_TEST_MAXPROCS=$GIT_LFS_TEST_MAXPROCS GIT_LFS_TEST_DIR="$GIT_LFS_TEST_DIR" SHUTDOWN_LFS="no" go run script/*.go -cmd integration "$@"
|
||||
|
@ -76,6 +76,7 @@ and the remote repository data in `test/remote`.
|
||||
* `SKIPCOMPILE=1` - This skips the Git LFS compilation step. Speeds up the
|
||||
tests when you're running the same test script multiple times without changing
|
||||
any Go code.
|
||||
* `GIT_LFS_USE_LEGACY_FILTER=1` - TODO
|
||||
|
||||
Also ensure that your `noproxy` environment variable contains `127.0.0.1` host,
|
||||
to allow git commands to reach the local Git server `lfstest-gitserver`.
|
||||
|
@ -46,6 +46,7 @@ begin_test "batch storage download causes retries"
|
||||
|
||||
pushd ..
|
||||
git \
|
||||
-c "filter.lfs.process=" \
|
||||
-c "filter.lfs.smudge=cat" \
|
||||
-c "filter.lfs.required=false" \
|
||||
clone "$GITSERVER/$reponame" "$reponame-assert"
|
||||
|
@ -2,7 +2,8 @@
|
||||
|
||||
. "test/testlib.sh"
|
||||
|
||||
envInitConfig='git config filter.lfs.smudge = "git-lfs smudge -- %f"
|
||||
envInitConfig='git config filter.lfs.process = "git-lfs filter"
|
||||
git config filter.lfs.smudge = "git-lfs smudge -- %f"
|
||||
git config filter.lfs.clean = "git-lfs clean -- %f"'
|
||||
|
||||
begin_test "env with no remote"
|
||||
@ -616,8 +617,6 @@ AccessUpload=none
|
||||
DownloadTransfers=basic
|
||||
UploadTransfers=basic
|
||||
%s
|
||||
git config filter.lfs.smudge = \"\"
|
||||
git config filter.lfs.clean = \"\"
|
||||
' "$(git lfs version)" "$(git version)" "$localwd" "$localgit" "$localgitstore" "$localmedia" "$tempdir" "$envVars")
|
||||
actual5=$(GIT_DIR=$gitDir GIT_WORK_TREE=a/b git lfs env)
|
||||
contains_same_elements "$expected5" "$actual5"
|
||||
|
@ -6,6 +6,7 @@ begin_test "install again"
|
||||
(
|
||||
set -e
|
||||
|
||||
# TODO: add process filter here
|
||||
smudge="$(git config filter.lfs.smudge)"
|
||||
clean="$(git config filter.lfs.clean)"
|
||||
|
||||
|
@ -47,6 +47,7 @@ begin_test "legacy download check causes retries"
|
||||
|
||||
pushd ..
|
||||
git \
|
||||
-c "filter.lfs.process=" \
|
||||
-c "filter.lfs.smudge=cat" \
|
||||
-c "filter.lfs.required=false" \
|
||||
clone "$GITSERVER/$reponame" "$reponame-assert"
|
||||
@ -108,6 +109,7 @@ begin_test "legacy storage download causes retries"
|
||||
|
||||
pushd ..
|
||||
git \
|
||||
-c "filter.lfs.process=" \
|
||||
-c "filter.lfs.smudge=cat" \
|
||||
-c "filter.lfs.required=false" \
|
||||
clone "$GITSERVER/$reponame" "$reponame-assert"
|
||||
|
@ -6,6 +6,7 @@ begin_test "uninstall outside repository"
|
||||
(
|
||||
set -e
|
||||
|
||||
# TODO: add process filter here
|
||||
smudge="$(git config filter.lfs.smudge)"
|
||||
clean="$(git config filter.lfs.clean)"
|
||||
|
||||
|
@ -3,7 +3,8 @@
|
||||
. "test/testlib.sh"
|
||||
|
||||
ensure_git_version_isnt $VERSION_LOWER "2.5.0"
|
||||
envInitConfig='git config filter.lfs.smudge = "git-lfs smudge -- %f"
|
||||
envInitConfig='git config filter.lfs.process = "git-lfs filter"
|
||||
git config filter.lfs.smudge = "git-lfs smudge -- %f"
|
||||
git config filter.lfs.clean = "git-lfs clean -- %f"'
|
||||
|
||||
begin_test "git worktree"
|
||||
|
@ -175,7 +175,7 @@ wait_for_file() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# setup_remote_repo intializes a bare Git repository that is accessible through
|
||||
# setup_remote_repo initializes a bare Git repository that is accessible through
|
||||
# the test Git server. The `pwd` is set to the repository's directory, in case
|
||||
# further commands need to be run. This server is running for every test in a
|
||||
# script/integration run, so every test file should setup its own remote
|
||||
@ -314,7 +314,12 @@ setup() {
|
||||
git config --global user.email "git-lfs@example.com"
|
||||
git config --global http.sslcainfo "$LFS_CERT_FILE"
|
||||
|
||||
grep "git-lfs clean" "$REMOTEDIR/home/.gitconfig" > /dev/null || {
|
||||
if [ "$GIT_LFS_USE_LEGACY_FILTER" == "1" ]; then
|
||||
FILTER="clean"
|
||||
else
|
||||
FILTER="filter"
|
||||
fi
|
||||
grep "git-lfs $FILTER" "$REMOTEDIR/home/.gitconfig" > /dev/null || {
|
||||
echo "global git config should be set in $REMOTEDIR/home"
|
||||
ls -al "$REMOTEDIR/home"
|
||||
exit 1
|
||||
|
Loading…
Reference in New Issue
Block a user