2015-07-30 02:37:31 +00:00
|
|
|
#!/usr/bin/env bash
|
2015-06-07 15:18:58 +00:00
|
|
|
|
2018-07-10 18:48:02 +00:00
|
|
|
. "$(dirname "$0")/testlib.sh"
|
2015-06-07 15:18:58 +00:00
|
|
|
|
|
|
|
begin_test "fsck default"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-default"
|
|
|
|
git init $reponame
|
|
|
|
cd $reponame
|
|
|
|
|
|
|
|
# Create a commit with some files tracked by git-lfs
|
|
|
|
git lfs track *.dat
|
|
|
|
echo "test data" > a.dat
|
|
|
|
echo "test data 2" > b.dat
|
|
|
|
git add .gitattributes *.dat
|
|
|
|
git commit -m "first commit"
|
|
|
|
|
|
|
|
[ "Git LFS fsck OK" = "$(git lfs fsck)" ]
|
|
|
|
|
|
|
|
aOid=$(git log --patch a.dat | grep "^+oid" | cut -d ":" -f 2)
|
|
|
|
aOid12=$(echo $aOid | cut -b 1-2)
|
|
|
|
aOid34=$(echo $aOid | cut -b 3-4)
|
2016-10-08 14:59:17 +00:00
|
|
|
if [ "$aOid" != "$(calc_oid_file .git/lfs/objects/$aOid12/$aOid34/$aOid)" ]; then
|
2015-06-07 15:18:58 +00:00
|
|
|
echo "oid for a.dat does not match"
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
bOid=$(git log --patch b.dat | grep "^+oid" | cut -d ":" -f 2)
|
|
|
|
bOid12=$(echo $bOid | cut -b 1-2)
|
|
|
|
bOid34=$(echo $bOid | cut -b 3-4)
|
2016-10-08 14:59:17 +00:00
|
|
|
if [ "$bOid" != "$(calc_oid_file .git/lfs/objects/$bOid12/$bOid34/$bOid)" ]; then
|
2015-06-07 15:18:58 +00:00
|
|
|
echo "oid for b.dat does not match"
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
echo "CORRUPTION" >> .git/lfs/objects/$aOid12/$aOid34/$aOid
|
|
|
|
|
2019-08-30 19:19:24 +00:00
|
|
|
moved=$(canonical_path "$TRASHDIR/$reponame/.git/lfs/bad")
|
2021-04-19 19:50:06 +00:00
|
|
|
expected="$(printf 'objects: corruptObject: a.dat (%s) is corrupt
|
|
|
|
objects: repair: moving corrupt objects to %s' "$aOid" "$moved")"
|
2015-06-07 15:18:58 +00:00
|
|
|
[ "$expected" = "$(git lfs fsck)" ]
|
|
|
|
|
2016-11-29 20:56:47 +00:00
|
|
|
[ -e ".git/lfs/bad/$aOid" ]
|
|
|
|
[ ! -e ".git/lfs/objects/$aOid12/$aOid34/$aOid" ]
|
|
|
|
[ "$bOid" = "$(calc_oid_file .git/lfs/objects/$bOid12/$bOid34/$bOid)" ]
|
2015-06-07 15:18:58 +00:00
|
|
|
)
|
|
|
|
end_test
|
|
|
|
|
|
|
|
begin_test "fsck dry run"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-dry-run"
|
|
|
|
git init $reponame
|
|
|
|
cd $reponame
|
|
|
|
|
|
|
|
# Create a commit with some files tracked by git-lfs
|
|
|
|
git lfs track *.dat
|
|
|
|
echo "test data" > a.dat
|
|
|
|
echo "test data 2" > b.dat
|
|
|
|
git add .gitattributes *.dat
|
|
|
|
git commit -m "first commit"
|
|
|
|
|
|
|
|
[ "Git LFS fsck OK" = "$(git lfs fsck --dry-run)" ]
|
|
|
|
|
|
|
|
aOid=$(git log --patch a.dat | grep "^+oid" | cut -d ":" -f 2)
|
|
|
|
aOid12=$(echo $aOid | cut -b 1-2)
|
|
|
|
aOid34=$(echo $aOid | cut -b 3-4)
|
2016-10-08 14:59:17 +00:00
|
|
|
if [ "$aOid" != "$(calc_oid_file .git/lfs/objects/$aOid12/$aOid34/$aOid)" ]; then
|
2015-06-07 15:18:58 +00:00
|
|
|
echo "oid for a.dat does not match"
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
bOid=$(git log --patch b.dat | grep "^+oid" | cut -d ":" -f 2)
|
|
|
|
bOid12=$(echo $bOid | cut -b 1-2)
|
|
|
|
bOid34=$(echo $bOid | cut -b 3-4)
|
2016-10-08 14:59:17 +00:00
|
|
|
if [ "$bOid" != "$(calc_oid_file .git/lfs/objects/$bOid12/$bOid34/$bOid)" ]; then
|
2015-06-07 15:18:58 +00:00
|
|
|
echo "oid for b.dat does not match"
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
echo "CORRUPTION" >> .git/lfs/objects/$aOid12/$aOid34/$aOid
|
|
|
|
|
2021-04-19 19:50:06 +00:00
|
|
|
[ "objects: corruptObject: a.dat ($aOid) is corrupt" = "$(git lfs fsck --dry-run)" ]
|
2015-06-07 15:18:58 +00:00
|
|
|
|
2016-10-08 14:59:17 +00:00
|
|
|
if [ "$aOid" = "$(calc_oid_file .git/lfs/objects/$aOid12/$aOid34/$aOid)" ]; then
|
2015-06-07 15:18:58 +00:00
|
|
|
echo "oid for a.dat still matches match"
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
2016-10-08 14:59:17 +00:00
|
|
|
if [ "$bOid" != "$(calc_oid_file .git/lfs/objects/$bOid12/$bOid34/$bOid)" ]; then
|
2015-06-07 15:18:58 +00:00
|
|
|
echo "oid for b.dat does not match"
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
)
|
|
|
|
end_test
|
2015-09-08 15:29:53 +00:00
|
|
|
|
2019-09-10 18:20:39 +00:00
|
|
|
begin_test "fsck does not fail with shell characters in paths"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
mkdir '[[path]]'
|
|
|
|
cd '[[path]]'
|
|
|
|
reponame="fsck-shell-paths"
|
|
|
|
git init $reponame
|
|
|
|
cd $reponame
|
|
|
|
|
|
|
|
# Create a commit with some files tracked by git-lfs
|
|
|
|
git lfs track *.dat
|
|
|
|
echo "test data" > a.dat
|
|
|
|
echo "test data 2" > b.dat
|
|
|
|
git add .gitattributes *.dat
|
|
|
|
git commit -m "first commit"
|
|
|
|
|
|
|
|
# Verify that the pack code handles glob patterns properly.
|
|
|
|
git gc --aggressive --prune=now
|
|
|
|
|
|
|
|
[ "Git LFS fsck OK" = "$(git lfs fsck)" ]
|
|
|
|
)
|
|
|
|
end_test
|
|
|
|
|
2015-09-08 15:29:53 +00:00
|
|
|
begin_test "fsck: outside git repository"
|
|
|
|
(
|
2015-09-08 16:20:52 +00:00
|
|
|
set +e
|
|
|
|
git lfs fsck 2>&1 > fsck.log
|
|
|
|
res=$?
|
2015-09-08 15:29:53 +00:00
|
|
|
set -e
|
2022-02-07 19:25:06 +00:00
|
|
|
|
2015-09-08 20:12:51 +00:00
|
|
|
if [ "$res" = "0" ]; then
|
|
|
|
echo "Passes because $GIT_LFS_TEST_DIR is unset."
|
|
|
|
exit 0
|
|
|
|
fi
|
2015-09-08 16:20:52 +00:00
|
|
|
[ "$res" = "128" ]
|
2022-01-29 00:44:16 +00:00
|
|
|
grep "Not in a Git repository" fsck.log
|
2015-09-08 15:29:53 +00:00
|
|
|
)
|
|
|
|
end_test
|
2021-06-10 16:21:34 +00:00
|
|
|
|
t/t-{attributes,fsck}.sh: test macro support
In commit f4b8938fd60dfb705d83a62f153acd59fe067a51 of PR #3391 the
t/t-attributes.sh test script was introduced with its initial "macros"
test, which validates that the "git lfs track" command is able to parse
macro attribute definitions in the top-level .gitattributes file and
resolve references to those macros in the same file. It also confirms
that the command does not accept macro definitions in .gitattributes
files in subdirectories, as Git does not accept these either.
However, Git does resolve macro attribute references from .gitattributes
files in subdirectories, so long as they refer to macro attributes
defined in the top-level .gitattributes (or one of the other files where
definitions are accepted, such as the .git/info/attributes file). But
the "git lfs track" command at present does not resolve such references
consistently because it sorts the attributes files by path length and
then processes them strictly in that order, from longest to shortest.
Thus references to macro attributes defined in the top-level .gitattributes
file from other attributes files never succeed because the top-level file
is always parsed last (except for the global and system attributes files).
We therefore add a note to this effect in the "macros" test to explain
why we do not test valid macro attribute references in a .gitattributes
file in a subdirectory.
(There is also an inconsistency in how "git lfs track" handles references
to macro attributes defined in the .git/info/attributes file, because if
the references appear in .gitattributes files whose full file path in the
repository is longer than ".git/info/attributes", then the references are
not resolved as these files are parsed before the .git/info/attributes one,
whereas references from other .gitattributes files are resolved.)
Separately, in commit 608bc8d53efe0bfc789dae4b934e18af69ebd8e5 of PR #4525
support for scanning the repository contents using the output of the
"git ls-tree" command was added to help enable the "git lfs fsck" to
search for invalid Git LFS pointer files. The GitScanner.ScanRefByTree()
method invokes a chain of functions, of which catFileBatchTreeForPointers()
reads Git blob metadata and examines each blob in turn to see if it is
a Git LFS pointer or a .gitattributes file, and if it is the latter it
reads and parses its contents, including macro attribute definitions if
the file is the top-level .gitattributes file.
We therefore add a "fsck detects invalid pointers with macro patterns"
test to the t/t-fsck.sh test script which validates the ability of the
"git lfs fsck" command to report as invalid pointers any files matching
patterns with a "filter=lfs" attribute defined by reference to a macro
attribute defined in the top-level .gitattributes file.
To do this we refactor the setup_invalid_pointers() helper function so
that we can reuse some of its code in a new, smaller function that just
creates invalid pointers.
However, we also add a note explaining that we can not yet test this
behaviour with a .gitattributes file whose parent directory sorts
before the top-level .gitattributes one in the output from "git ls-tree".
Because that command outputs its results sorted by filepath, a file such
as .dir/.gitattributes will be listed before the top-level .gitattributes
file, and so any macro attribute references from the .dir/.gitattributes
file to macro attributes defined in the top-level .gitattributes file
will not be resolved in the way that Git resolves them.
For now we defer resolution of this issue and the ones described regarding
the "git lfs track" command to the future.
2022-11-07 16:42:04 +00:00
|
|
|
create_invalid_pointers() {
|
|
|
|
valid="$1"
|
|
|
|
ext="${2:-dat}"
|
|
|
|
|
|
|
|
git cat-file blob ":$valid" | awk '{ sub(/$/, "\r"); print }' >"crlf.$ext"
|
|
|
|
base64 /dev/urandom | head -c 1025 >"large.$ext"
|
|
|
|
git \
|
|
|
|
-c "filter.lfs.process=" \
|
|
|
|
-c "filter.lfs.clean=cat" \
|
|
|
|
-c "filter.lfs.required=false" \
|
|
|
|
add "crlf.$ext" "large.$ext"
|
|
|
|
git commit -m "invalid pointers"
|
|
|
|
}
|
|
|
|
|
2021-06-10 18:02:43 +00:00
|
|
|
setup_invalid_pointers () {
|
2021-06-10 16:21:34 +00:00
|
|
|
git init $reponame
|
|
|
|
cd $reponame
|
|
|
|
|
|
|
|
# Create a commit with some files tracked by git-lfs
|
|
|
|
git lfs track *.dat
|
|
|
|
echo "test data" > a.dat
|
|
|
|
echo "test data 2" > b.dat
|
|
|
|
git add .gitattributes *.dat
|
2021-06-10 18:02:43 +00:00
|
|
|
git commit -m "first commit"
|
2021-06-10 16:21:34 +00:00
|
|
|
|
t/t-{attributes,fsck}.sh: test macro support
In commit f4b8938fd60dfb705d83a62f153acd59fe067a51 of PR #3391 the
t/t-attributes.sh test script was introduced with its initial "macros"
test, which validates that the "git lfs track" command is able to parse
macro attribute definitions in the top-level .gitattributes file and
resolve references to those macros in the same file. It also confirms
that the command does not accept macro definitions in .gitattributes
files in subdirectories, as Git does not accept these either.
However, Git does resolve macro attribute references from .gitattributes
files in subdirectories, so long as they refer to macro attributes
defined in the top-level .gitattributes (or one of the other files where
definitions are accepted, such as the .git/info/attributes file). But
the "git lfs track" command at present does not resolve such references
consistently because it sorts the attributes files by path length and
then processes them strictly in that order, from longest to shortest.
Thus references to macro attributes defined in the top-level .gitattributes
file from other attributes files never succeed because the top-level file
is always parsed last (except for the global and system attributes files).
We therefore add a note to this effect in the "macros" test to explain
why we do not test valid macro attribute references in a .gitattributes
file in a subdirectory.
(There is also an inconsistency in how "git lfs track" handles references
to macro attributes defined in the .git/info/attributes file, because if
the references appear in .gitattributes files whose full file path in the
repository is longer than ".git/info/attributes", then the references are
not resolved as these files are parsed before the .git/info/attributes one,
whereas references from other .gitattributes files are resolved.)
Separately, in commit 608bc8d53efe0bfc789dae4b934e18af69ebd8e5 of PR #4525
support for scanning the repository contents using the output of the
"git ls-tree" command was added to help enable the "git lfs fsck" to
search for invalid Git LFS pointer files. The GitScanner.ScanRefByTree()
method invokes a chain of functions, of which catFileBatchTreeForPointers()
reads Git blob metadata and examines each blob in turn to see if it is
a Git LFS pointer or a .gitattributes file, and if it is the latter it
reads and parses its contents, including macro attribute definitions if
the file is the top-level .gitattributes file.
We therefore add a "fsck detects invalid pointers with macro patterns"
test to the t/t-fsck.sh test script which validates the ability of the
"git lfs fsck" command to report as invalid pointers any files matching
patterns with a "filter=lfs" attribute defined by reference to a macro
attribute defined in the top-level .gitattributes file.
To do this we refactor the setup_invalid_pointers() helper function so
that we can reuse some of its code in a new, smaller function that just
creates invalid pointers.
However, we also add a note explaining that we can not yet test this
behaviour with a .gitattributes file whose parent directory sorts
before the top-level .gitattributes one in the output from "git ls-tree".
Because that command outputs its results sorted by filepath, a file such
as .dir/.gitattributes will be listed before the top-level .gitattributes
file, and so any macro attribute references from the .dir/.gitattributes
file to macro attributes defined in the top-level .gitattributes file
will not be resolved in the way that Git resolves them.
For now we defer resolution of this issue and the ones described regarding
the "git lfs track" command to the future.
2022-11-07 16:42:04 +00:00
|
|
|
create_invalid_pointers "a.dat"
|
2021-06-10 18:02:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
begin_test "fsck detects invalid pointers"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-pointers"
|
|
|
|
setup_invalid_pointers
|
2021-06-10 16:21:34 +00:00
|
|
|
|
|
|
|
set +e
|
|
|
|
git lfs fsck >test.log 2>&1
|
|
|
|
RET=$?
|
|
|
|
git lfs fsck --pointers >>test.log 2>&1
|
|
|
|
RET2=$?
|
|
|
|
set -e
|
|
|
|
|
|
|
|
[ "$RET" -eq 1 ]
|
|
|
|
[ "$RET2" -eq 1 ]
|
|
|
|
[ $(grep -c 'pointer: nonCanonicalPointer: Pointer.*was not canonical' test.log) -eq 2 ]
|
|
|
|
[ $(grep -c 'pointer: unexpectedGitObject: "large.dat".*should have been a pointer but was not' test.log) -eq 2 ]
|
|
|
|
)
|
|
|
|
end_test
|
2021-06-10 18:02:43 +00:00
|
|
|
|
t/t-{attributes,fsck}.sh: test macro support
In commit f4b8938fd60dfb705d83a62f153acd59fe067a51 of PR #3391 the
t/t-attributes.sh test script was introduced with its initial "macros"
test, which validates that the "git lfs track" command is able to parse
macro attribute definitions in the top-level .gitattributes file and
resolve references to those macros in the same file. It also confirms
that the command does not accept macro definitions in .gitattributes
files in subdirectories, as Git does not accept these either.
However, Git does resolve macro attribute references from .gitattributes
files in subdirectories, so long as they refer to macro attributes
defined in the top-level .gitattributes (or one of the other files where
definitions are accepted, such as the .git/info/attributes file). But
the "git lfs track" command at present does not resolve such references
consistently because it sorts the attributes files by path length and
then processes them strictly in that order, from longest to shortest.
Thus references to macro attributes defined in the top-level .gitattributes
file from other attributes files never succeed because the top-level file
is always parsed last (except for the global and system attributes files).
We therefore add a note to this effect in the "macros" test to explain
why we do not test valid macro attribute references in a .gitattributes
file in a subdirectory.
(There is also an inconsistency in how "git lfs track" handles references
to macro attributes defined in the .git/info/attributes file, because if
the references appear in .gitattributes files whose full file path in the
repository is longer than ".git/info/attributes", then the references are
not resolved as these files are parsed before the .git/info/attributes one,
whereas references from other .gitattributes files are resolved.)
Separately, in commit 608bc8d53efe0bfc789dae4b934e18af69ebd8e5 of PR #4525
support for scanning the repository contents using the output of the
"git ls-tree" command was added to help enable the "git lfs fsck" to
search for invalid Git LFS pointer files. The GitScanner.ScanRefByTree()
method invokes a chain of functions, of which catFileBatchTreeForPointers()
reads Git blob metadata and examines each blob in turn to see if it is
a Git LFS pointer or a .gitattributes file, and if it is the latter it
reads and parses its contents, including macro attribute definitions if
the file is the top-level .gitattributes file.
We therefore add a "fsck detects invalid pointers with macro patterns"
test to the t/t-fsck.sh test script which validates the ability of the
"git lfs fsck" command to report as invalid pointers any files matching
patterns with a "filter=lfs" attribute defined by reference to a macro
attribute defined in the top-level .gitattributes file.
To do this we refactor the setup_invalid_pointers() helper function so
that we can reuse some of its code in a new, smaller function that just
creates invalid pointers.
However, we also add a note explaining that we can not yet test this
behaviour with a .gitattributes file whose parent directory sorts
before the top-level .gitattributes one in the output from "git ls-tree".
Because that command outputs its results sorted by filepath, a file such
as .dir/.gitattributes will be listed before the top-level .gitattributes
file, and so any macro attribute references from the .dir/.gitattributes
file to macro attributes defined in the top-level .gitattributes file
will not be resolved in the way that Git resolves them.
For now we defer resolution of this issue and the ones described regarding
the "git lfs track" command to the future.
2022-11-07 16:42:04 +00:00
|
|
|
begin_test "fsck detects invalid pointers with macro patterns"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-pointers-macros"
|
|
|
|
git init $reponame
|
|
|
|
cd $reponame
|
|
|
|
|
|
|
|
printf '[attr]lfs filter=lfs diff=lfs merge=lfs -text\n*.dat lfs\n' \
|
|
|
|
>.gitattributes
|
|
|
|
echo "test data" >a.dat
|
|
|
|
mkdir dir
|
|
|
|
printf '*.bin lfs\n' >dir/.gitattributes
|
|
|
|
git add .gitattributes a.dat dir
|
|
|
|
git commit -m "first commit"
|
|
|
|
|
|
|
|
create_invalid_pointers "a.dat"
|
|
|
|
|
|
|
|
cd dir
|
|
|
|
create_invalid_pointers "a.dat" "bin"
|
|
|
|
cd ..
|
|
|
|
|
|
|
|
# NOTE: We should also create a .dir directory with the same files as
|
|
|
|
# as in the dir/ directory, and confirm those .dir/*.bin files are
|
|
|
|
# reported by "git lfs fsck" as well. However, at the moment
|
|
|
|
# "git lfs fsck" will not resolve a macro attribute reference
|
|
|
|
# in .dir/.gitattributes because it sorts that file before
|
|
|
|
# .gitattributes and then processes them in that order.
|
|
|
|
|
|
|
|
set +e
|
|
|
|
git lfs fsck >test.log 2>&1
|
|
|
|
RET=$?
|
|
|
|
git lfs fsck --pointers >>test.log 2>&1
|
|
|
|
RET2=$?
|
|
|
|
set -e
|
|
|
|
|
|
|
|
[ "$RET" -eq 1 ]
|
|
|
|
[ "$RET2" -eq 1 ]
|
|
|
|
[ $(grep -c 'pointer: nonCanonicalPointer: Pointer.*was not canonical' test.log) -eq 4 ]
|
|
|
|
[ $(grep -c 'pointer: unexpectedGitObject: "large.dat".*should have been a pointer but was not' test.log) -eq 2 ]
|
|
|
|
[ $(grep -c 'pointer: unexpectedGitObject: "dir/large.bin".*should have been a pointer but was not' test.log) -eq 2 ]
|
|
|
|
)
|
|
|
|
end_test
|
|
|
|
|
2021-10-14 17:56:17 +00:00
|
|
|
begin_test "fsck detects invalid pointers with GIT_OBJECT_DIRECTORY"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-pointers-object-directory"
|
|
|
|
setup_invalid_pointers
|
|
|
|
|
|
|
|
head=$(git rev-parse HEAD)
|
|
|
|
objdir="$(lfstest-realpath .git/objects)"
|
|
|
|
cd ..
|
|
|
|
git init "$reponame-2"
|
|
|
|
gitdir="$(lfstest-realpath "$reponame-2/.git")"
|
|
|
|
GIT_WORK_TREE="$reponame-2" GIT_DIR="$gitdir" GIT_OBJECT_DIRECTORY="$objdir" git update-ref refs/heads/main "$head"
|
2022-02-07 19:25:06 +00:00
|
|
|
|
2021-10-14 17:56:17 +00:00
|
|
|
set +e
|
|
|
|
GIT_WORK_TREE="$reponame-2" GIT_DIR="$gitdir" GIT_OBJECT_DIRECTORY="$objdir" git lfs fsck --pointers >test.log 2>&1
|
|
|
|
RET=$?
|
|
|
|
set -e
|
|
|
|
|
|
|
|
[ "$RET" -eq 1 ]
|
|
|
|
grep 'pointer: nonCanonicalPointer: Pointer.*was not canonical' test.log
|
|
|
|
grep 'pointer: unexpectedGitObject: "large.dat".*should have been a pointer but was not' test.log
|
|
|
|
)
|
|
|
|
end_test
|
|
|
|
|
2021-10-18 19:09:40 +00:00
|
|
|
begin_test "fsck does not detect invalid pointers with no LFS objects"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-pointers-none"
|
|
|
|
git init "$reponame"
|
|
|
|
cd "$reponame"
|
|
|
|
|
|
|
|
echo "# README" > README.md
|
|
|
|
git add README.md
|
|
|
|
git commit -m "Add README"
|
|
|
|
|
|
|
|
git lfs fsck
|
|
|
|
git lfs fsck --pointers
|
|
|
|
)
|
|
|
|
end_test
|
|
|
|
|
2021-10-20 20:39:20 +00:00
|
|
|
begin_test "fsck does not detect invalid pointers with symlinks"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-pointers-symlinks"
|
|
|
|
git init "$reponame"
|
|
|
|
cd "$reponame"
|
|
|
|
|
|
|
|
git lfs track '*.dat'
|
|
|
|
|
|
|
|
echo "# Test" > a.dat
|
|
|
|
ln -s a.dat b.dat
|
|
|
|
git add .gitattributes *.dat
|
|
|
|
git commit -m "Add files"
|
|
|
|
|
|
|
|
git lfs fsck
|
|
|
|
git lfs fsck --pointers
|
|
|
|
)
|
|
|
|
end_test
|
|
|
|
|
2021-10-21 13:26:37 +00:00
|
|
|
begin_test "fsck does not detect invalid pointers with negated patterns"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-pointers-none"
|
|
|
|
git init "$reponame"
|
|
|
|
cd "$reponame"
|
|
|
|
|
|
|
|
cat > .gitattributes <<EOF
|
|
|
|
*.dat filter=lfs diff=lfs merge=lfs -text
|
|
|
|
b.dat !filter !diff !merge text
|
|
|
|
EOF
|
|
|
|
|
|
|
|
echo "# Test" > a.dat
|
|
|
|
cp a.dat b.dat
|
|
|
|
git add .gitattributes *.dat
|
|
|
|
git commit -m "Add files"
|
|
|
|
|
|
|
|
git lfs fsck
|
|
|
|
git lfs fsck --pointers
|
|
|
|
)
|
|
|
|
end_test
|
|
|
|
|
git/gitattr,t: handle unspecified macro attributes
In commit 1ff52542da97d6418689a5528112b57e37536014 of PR #3391 we
introduced the MacroProcessor type and methods to support the use of
macro attributes in .gitattributes files.
However, we do not currently support the case where a macro attributes
is specified with a "!" prefix, which Git handles by setting all
attributes defined by the macro attribute back to the unspecified state.
(Note that the "-" prefix is not supported by Git for macro attributes,
only the "!" one.)
To mimic the same behaviour in Git LFS we add a check for a macro
attribute with its Unspecified bool set to "true", and when this is
detected we iterate through the set of attributes defined by the macro
attribute and set them all to the same unspecified state.
We also add tests to confirm this new handling works as expected, both
a new Go test and a new "fsck does not detect invalid pointers with
negated macro patterns" test in t/t-fsck.sh that will not succeed without
the changes to the MacroProcessor in this commit. Without these changes,
any patterns that reference a macro attribute with the "!" prefix are not
processed as making the macro's attributes all unspecified again, and so
non-pointer files matching those patterns are reported as invalid Git LFS
pointers.
In the new test in t/t-fsck.sh we include comments describing how the
"git lfs fsck" command currently processes .gitattributes files in
the order returned by "git ls-tree", and so a .gitattributes file in
a subdirectory such as .dir/ will be parsed before the top-level
.gitattributes one because it appears first in the "git ls-tree" output.
The result is that any macro attribute references in the
.dir/.gitattributes file will not be resolved properly, and so our
test succeeds but not quite for the right reasons.
We also add a new "macros with unspecified flag" test in the
t/t-attributes.sh test script, but this test ultimately is only a
placeholder as it can not actually test that the "git lfs track" command
will not overwrite a pattern in a .gitattributes file in a subdirectory
if it references a macro attribute defined in the top-level .gitattributes
file and the reference has the "!" prefix. This is due to the fact that
the "git lfs track" command parses .gitattributes files in the order
of the length of their full paths, from longest to shortest, and so
macro attribute references can not be resolved except within the top-level
.gitattributes file (with some caveats regarding the .git/info/attributes
file and the global and system attributes files).
For now we defer resolution of both this issue and the one described
regarding the "git lfs fsck" command to the future.
2022-11-07 16:43:37 +00:00
|
|
|
begin_test "fsck does not detect invalid pointers with negated macro patterns"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-pointers-macros-none"
|
|
|
|
git init "$reponame"
|
|
|
|
cd "$reponame"
|
|
|
|
|
|
|
|
printf '[attr]lfs filter=lfs diff=lfs merge=lfs -text\n*.dat lfs\nb.dat !lfs\n' \
|
|
|
|
>.gitattributes
|
|
|
|
echo "test data" >a.dat
|
|
|
|
cp a.dat b.dat
|
|
|
|
mkdir dir .dir
|
|
|
|
printf '*.dat !lfs\n' >dir/.gitattributes
|
|
|
|
cp b.dat dir
|
|
|
|
printf '*.dat !lfs\n' >.dir/.gitattributes
|
|
|
|
cp b.dat .dir
|
|
|
|
git add .gitattributes *.dat dir .dir
|
|
|
|
git commit -m "first commit"
|
|
|
|
|
|
|
|
# NOTE: The "git lfs fsck" command exempts the .dir/b.dat file from the
|
|
|
|
# *.dat pattern from the top-level .gitattributes and so permits
|
|
|
|
# it as a valid non-pointer file; however, it permits it for a
|
|
|
|
# different reason than the dir/b.dat file, because it processes
|
|
|
|
# the .dir/.gitattributes file before the .gitattributes one
|
|
|
|
# and does not recognize the "!lfs" macro attribute reference until
|
|
|
|
# after it has processed .gitattributes. Ideally both the dir/
|
|
|
|
# and .dir/ directories should be processed identically.
|
|
|
|
|
|
|
|
git lfs fsck
|
|
|
|
git lfs fsck --pointers
|
|
|
|
)
|
|
|
|
end_test
|
|
|
|
|
2022-02-06 04:29:36 +00:00
|
|
|
setup_invalid_objects () {
|
|
|
|
git init $reponame
|
|
|
|
cd $reponame
|
|
|
|
|
|
|
|
# Create a commit with some files tracked by git-lfs
|
|
|
|
git lfs track *.dat
|
|
|
|
echo "test data" > a.dat
|
|
|
|
echo "test data 2" > b.dat
|
2022-04-19 06:56:59 +00:00
|
|
|
mkdir foo
|
|
|
|
echo "test test 3" > foo/a.dat
|
|
|
|
echo "test data 4" > foo/b.dat
|
|
|
|
git add .gitattributes *.dat foo
|
2022-02-06 04:29:36 +00:00
|
|
|
git commit -m "first commit"
|
|
|
|
|
|
|
|
oid1=$(calc_oid_file a.dat)
|
|
|
|
oid2=$(calc_oid_file b.dat)
|
2022-04-19 06:56:59 +00:00
|
|
|
oid3=$(calc_oid_file foo/a.dat)
|
|
|
|
oid4=$(calc_oid_file foo/b.dat)
|
2022-02-06 04:29:36 +00:00
|
|
|
echo "CORRUPTION" >>".git/lfs/objects/${oid1:0:2}/${oid1:2:2}/$oid1"
|
|
|
|
rm ".git/lfs/objects/${oid2:0:2}/${oid2:2:2}/$oid2"
|
2022-04-19 06:56:59 +00:00
|
|
|
echo "CORRUPTION" >>".git/lfs/objects/${oid3:0:2}/${oid3:2:2}/$oid3"
|
|
|
|
rm ".git/lfs/objects/${oid4:0:2}/${oid4:2:2}/$oid4"
|
2022-02-06 04:29:36 +00:00
|
|
|
}
|
2022-02-07 19:25:06 +00:00
|
|
|
|
2022-02-06 04:29:36 +00:00
|
|
|
begin_test "fsck detects invalid objects"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-objects"
|
|
|
|
setup_invalid_objects
|
|
|
|
|
|
|
|
set +e
|
|
|
|
git lfs fsck >test.log 2>&1
|
|
|
|
RET=$?
|
|
|
|
set -e
|
|
|
|
|
2022-02-06 06:22:39 +00:00
|
|
|
[ "$RET" -eq 1 ]
|
2022-02-06 04:29:36 +00:00
|
|
|
[ $(grep -c 'objects: corruptObject: a.dat (.*) is corrupt' test.log) -eq 1 ]
|
|
|
|
[ $(grep -c 'objects: openError: b.dat (.*) could not be checked: .*' test.log) -eq 1 ]
|
2022-04-19 06:56:59 +00:00
|
|
|
[ $(grep -c 'objects: corruptObject: foo/a.dat (.*) is corrupt' test.log) -eq 1 ]
|
|
|
|
[ $(grep -c 'objects: openError: foo/b.dat (.*) could not be checked: .*' test.log) -eq 1 ]
|
2022-02-06 04:29:36 +00:00
|
|
|
[ $(grep -c 'objects: repair: moving corrupt objects to .*' test.log) -eq 1 ]
|
|
|
|
|
|
|
|
cd ..
|
|
|
|
rm -rf $reponame
|
|
|
|
setup_invalid_objects
|
|
|
|
|
|
|
|
set +e
|
|
|
|
git lfs fsck --objects >test.log 2>&1
|
|
|
|
RET=$?
|
|
|
|
set -e
|
|
|
|
|
2022-02-06 06:22:39 +00:00
|
|
|
[ "$RET" -eq 1 ]
|
2022-02-06 04:29:36 +00:00
|
|
|
[ $(grep -c 'objects: corruptObject: a.dat (.*) is corrupt' test.log) -eq 1 ]
|
|
|
|
[ $(grep -c 'objects: openError: b.dat (.*) could not be checked: .*' test.log) -eq 1 ]
|
2022-04-19 06:56:59 +00:00
|
|
|
[ $(grep -c 'objects: corruptObject: foo/a.dat (.*) is corrupt' test.log) -eq 1 ]
|
|
|
|
[ $(grep -c 'objects: openError: foo/b.dat (.*) could not be checked: .*' test.log) -eq 1 ]
|
2022-02-06 04:29:36 +00:00
|
|
|
[ $(grep -c 'objects: repair: moving corrupt objects to .*' test.log) -eq 1 ]
|
|
|
|
)
|
|
|
|
end_test
|
|
|
|
|
commands,t: gitignore matching for fetch filters
The "lfs.fetchinclude" and "lfs.fetchexclude" Git configuration
options, if set, are used to control the action of a number of Git
LFS commands. Since PR #4556, the "git lfs clone", "git lfs fetch",
and "git lfs pull" commands have strictly applied gitignore(5)-style
matching rules to these configuration options.
However, other commands including "git lfs filter-process" and
"git lfs smudge" now apply gitattributes(5)-style matching
rules to these same configuration options, leading to confusion.
We therefore revise all remaining uses of these configuration
options to also use gitignore-style matching rules.
We also add new tests for the "git lfs filter-process" and "git lfs
fsck" commands and adjust or expand existing tests for the "git lfs
prune" and "git lfs smudge" commands in order to confirm that
gitignore-style matching is used for all of them. These new and
updated tests fail if gitattributes-style matching is used instead.
(Note that the "git lfs migrate" command does not require any changes
because it does not read the "lfs.fetch*" configuration options.
Instead, it supplies a "false" value for the "useFetchOptions" flag
to the determineIncludeExcludePaths() function, so any "lfs.fetch*"
configuration values are ignored. This is significant because
"git lfs migrate" deliberately uses gitattributes-style matching
for any path patterns supplied via its -I/-X command-line arguments,
unlike all other commands that accept -I/-X arguments as overrides
for the "lfs.fetch*" configuration options.)
2022-04-18 07:24:53 +00:00
|
|
|
begin_test "fsck detects invalid objects except in excluded paths"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-objects-exclude"
|
|
|
|
setup_invalid_objects
|
|
|
|
|
|
|
|
# We need to prevent MSYS from rewriting /foo into a Windows path.
|
|
|
|
MSYS_NO_PATHCONV=1 git config "lfs.fetchexclude" "/foo"
|
|
|
|
|
|
|
|
set +e
|
|
|
|
git lfs fsck >test.log 2>&1
|
|
|
|
RET=$?
|
|
|
|
set -e
|
|
|
|
|
|
|
|
[ "$RET" -eq 1 ]
|
|
|
|
[ $(grep -c 'objects: corruptObject: a.dat (.*) is corrupt' test.log) -eq 1 ]
|
|
|
|
[ $(grep -c 'objects: openError: b.dat (.*) could not be checked: .*' test.log) -eq 1 ]
|
|
|
|
[ $(grep -c 'objects: corruptObject: foo/a.dat (.*) is corrupt' test.log) -eq 0 ]
|
|
|
|
[ $(grep -c 'objects: openError: foo/b.dat (.*) could not be checked: .*' test.log) -eq 0 ]
|
|
|
|
[ $(grep -c 'objects: repair: moving corrupt objects to .*' test.log) -eq 1 ]
|
|
|
|
|
|
|
|
cd ..
|
|
|
|
rm -rf $reponame
|
|
|
|
setup_invalid_objects
|
|
|
|
|
|
|
|
# We need to prevent MSYS from rewriting /foo into a Windows path.
|
|
|
|
MSYS_NO_PATHCONV=1 git config "lfs.fetchexclude" "/foo"
|
|
|
|
|
|
|
|
set +e
|
|
|
|
git lfs fsck --objects >test.log 2>&1
|
|
|
|
RET=$?
|
|
|
|
set -e
|
|
|
|
|
|
|
|
[ "$RET" -eq 1 ]
|
|
|
|
[ $(grep -c 'objects: corruptObject: a.dat (.*) is corrupt' test.log) -eq 1 ]
|
|
|
|
[ $(grep -c 'objects: openError: b.dat (.*) could not be checked: .*' test.log) -eq 1 ]
|
|
|
|
[ $(grep -c 'objects: corruptObject: foo/a.dat (.*) is corrupt' test.log) -eq 0 ]
|
|
|
|
[ $(grep -c 'objects: openError: foo/b.dat (.*) could not be checked: .*' test.log) -eq 0 ]
|
|
|
|
[ $(grep -c 'objects: repair: moving corrupt objects to .*' test.log) -eq 1 ]
|
|
|
|
)
|
|
|
|
end_test
|
|
|
|
|
2022-02-06 04:29:36 +00:00
|
|
|
begin_test "fsck does not detect invalid objects with no LFS objects"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-objects-none"
|
|
|
|
git init "$reponame"
|
|
|
|
cd "$reponame"
|
|
|
|
|
|
|
|
echo "# README" > README.md
|
|
|
|
git add README.md
|
|
|
|
git commit -m "Add README"
|
|
|
|
|
|
|
|
git lfs fsck
|
|
|
|
git lfs fsck --objects
|
|
|
|
)
|
|
|
|
end_test
|
|
|
|
|
2021-06-10 18:02:43 +00:00
|
|
|
begin_test "fsck operates on specified refs"
|
|
|
|
(
|
|
|
|
set -e
|
|
|
|
|
|
|
|
reponame="fsck-refs"
|
|
|
|
setup_invalid_pointers
|
|
|
|
|
|
|
|
git rm -f crlf.dat large.dat
|
commands,t: reverse ref args for fsck --objects
The "git lfs fsck --objects" command currently handles refs
expressed on the command line in two-dot range notation backwards;
this is due to the fact that we pass the first ref (identifying
everything to be excluded) as the "left" argument to the
gitscanner.ScanRefRange(), and the second ref as the "right"
argument. That would seem to be correct, except that in the
gitscanner methods, "left" is meant to define all the commits to
include, while "right" is meant to define the ones to exclude.
This terminology likely stems from the way refs are passed to the
pre-push hook, with the local one coming first (on the left), and
the remote one coming second. Since in that context we want to
exclude all the objects already on the remote, "right" (remote)
becomes the "exclude" ref, while "left" (local) becomes the
"include" one.
We can resolve this for the "git lfs fsck --objects" command
simply by reversing the two arguments. Note that we don't need
to reverse the arguments for the --pointers command option,
even though it passes them in the same "backwards" order to the
gitscanner.ScanRefRangeByTree() method, because that method
happens to reverse its "left" and "right" arguments when it calls
scanRefsByTree(). We will address this counter-intuitive design
in a subsequent commit.
We also expand the "fsck operates on specified refs" test so that
it explicitly checks both the --pointers and --objects options,
checking the former against a commit with bad Git LFS pointer
objects, and checking the latter against a different commit with
an object file with incorrect contents. This second new check
would fail without the reversal of the "left" and "right" arguments
to the gitscanner.ScanRefRange() method made in this commit.
2022-02-06 07:46:55 +00:00
|
|
|
echo "# Test" > new.dat
|
|
|
|
git add new.dat
|
2021-06-10 18:02:43 +00:00
|
|
|
git commit -m 'third commit'
|
|
|
|
|
|
|
|
git commit --allow-empty -m 'fourth commit'
|
|
|
|
|
|
|
|
# Should succeed. (HEAD and index).
|
commands,t: reverse ref args for fsck --objects
The "git lfs fsck --objects" command currently handles refs
expressed on the command line in two-dot range notation backwards;
this is due to the fact that we pass the first ref (identifying
everything to be excluded) as the "left" argument to the
gitscanner.ScanRefRange(), and the second ref as the "right"
argument. That would seem to be correct, except that in the
gitscanner methods, "left" is meant to define all the commits to
include, while "right" is meant to define the ones to exclude.
This terminology likely stems from the way refs are passed to the
pre-push hook, with the local one coming first (on the left), and
the remote one coming second. Since in that context we want to
exclude all the objects already on the remote, "right" (remote)
becomes the "exclude" ref, while "left" (local) becomes the
"include" one.
We can resolve this for the "git lfs fsck --objects" command
simply by reversing the two arguments. Note that we don't need
to reverse the arguments for the --pointers command option,
even though it passes them in the same "backwards" order to the
gitscanner.ScanRefRangeByTree() method, because that method
happens to reverse its "left" and "right" arguments when it calls
scanRefsByTree(). We will address this counter-intuitive design
in a subsequent commit.
We also expand the "fsck operates on specified refs" test so that
it explicitly checks both the --pointers and --objects options,
checking the former against a commit with bad Git LFS pointer
objects, and checking the latter against a different commit with
an object file with incorrect contents. This second new check
would fail without the reversal of the "left" and "right" arguments
to the gitscanner.ScanRefRange() method made in this commit.
2022-02-06 07:46:55 +00:00
|
|
|
|
2021-06-10 18:02:43 +00:00
|
|
|
git lfs fsck
|
|
|
|
git lfs fsck HEAD
|
|
|
|
git lfs fsck HEAD^^ && exit 1
|
|
|
|
git lfs fsck HEAD^
|
|
|
|
git lfs fsck HEAD^..HEAD
|
|
|
|
git lfs fsck HEAD^^^..HEAD && exit 1
|
|
|
|
git lfs fsck HEAD^^^..HEAD^ && exit 1
|
commands,t: reverse ref args for fsck --objects
The "git lfs fsck --objects" command currently handles refs
expressed on the command line in two-dot range notation backwards;
this is due to the fact that we pass the first ref (identifying
everything to be excluded) as the "left" argument to the
gitscanner.ScanRefRange(), and the second ref as the "right"
argument. That would seem to be correct, except that in the
gitscanner methods, "left" is meant to define all the commits to
include, while "right" is meant to define the ones to exclude.
This terminology likely stems from the way refs are passed to the
pre-push hook, with the local one coming first (on the left), and
the remote one coming second. Since in that context we want to
exclude all the objects already on the remote, "right" (remote)
becomes the "exclude" ref, while "left" (local) becomes the
"include" one.
We can resolve this for the "git lfs fsck --objects" command
simply by reversing the two arguments. Note that we don't need
to reverse the arguments for the --pointers command option,
even though it passes them in the same "backwards" order to the
gitscanner.ScanRefRangeByTree() method, because that method
happens to reverse its "left" and "right" arguments when it calls
scanRefsByTree(). We will address this counter-intuitive design
in a subsequent commit.
We also expand the "fsck operates on specified refs" test so that
it explicitly checks both the --pointers and --objects options,
checking the former against a commit with bad Git LFS pointer
objects, and checking the latter against a different commit with
an object file with incorrect contents. This second new check
would fail without the reversal of the "left" and "right" arguments
to the gitscanner.ScanRefRange() method made in this commit.
2022-02-06 07:46:55 +00:00
|
|
|
|
|
|
|
git lfs fsck --pointers HEAD^^^..HEAD^^ >test.log 2>&1 && exit 1
|
|
|
|
|
|
|
|
grep 'pointer: nonCanonicalPointer: Pointer.*was not canonical' test.log
|
|
|
|
grep 'pointer: unexpectedGitObject: "large.dat".*should have been a pointer but was not' test.log
|
|
|
|
|
|
|
|
oid=$(calc_oid_file new.dat)
|
|
|
|
echo "CORRUPTION" >>".git/lfs/objects/${oid:0:2}/${oid:2:2}/$oid"
|
|
|
|
|
|
|
|
git lfs fsck --objects HEAD^^..HEAD^ >test.log 2>&1 && exit 1
|
|
|
|
|
|
|
|
grep 'objects: corruptObject: new.dat (.*) is corrupt' test.log
|
|
|
|
grep 'objects: repair: moving corrupt objects to .*' test.log
|
|
|
|
|
2021-06-10 18:02:43 +00:00
|
|
|
# Make the result of the subshell a success.
|
|
|
|
true
|
|
|
|
)
|
|
|
|
end_test
|