rewrite fsck command to use gitscanner api
This commit is contained in:
risk danger olson
parent
3085dc32c7
commit
0569c1753c
@ -3,7 +3,6 @@ package commands
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@ -18,98 +17,6 @@ var (
|
||||
fsckDryRun bool
|
||||
)
|
||||
|
||||
func doFsck() (bool, error) {
|
||||
requireInRepo()
|
||||
|
||||
ref, err := git.CurrentRef()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// The LFS scanner methods return unexported *lfs.wrappedPointer objects.
|
||||
// All we care about is the pointer OID and file name
|
||||
pointerIndex, err := getPointersFromRef(ref.Sha)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
ok := true
|
||||
|
||||
for oid, name := range pointerIndex {
|
||||
path := lfs.LocalMediaPathReadOnly(oid)
|
||||
|
||||
Debug("Examining %v (%v)", name, path)
|
||||
|
||||
f, err := os.Open(path)
|
||||
if pErr, pOk := err.(*os.PathError); pOk {
|
||||
Print("Object %s (%s) could not be checked: %s", name, oid, pErr.Err)
|
||||
ok = false
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
oidHash := sha256.New()
|
||||
_, err = io.Copy(oidHash, f)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
recalculatedOid := hex.EncodeToString(oidHash.Sum(nil))
|
||||
if recalculatedOid != oid {
|
||||
ok = false
|
||||
Print("Object %s (%s) is corrupt", name, oid)
|
||||
if fsckDryRun {
|
||||
continue
|
||||
}
|
||||
|
||||
badDir := filepath.Join(config.LocalGitStorageDir, "lfs", "bad")
|
||||
if err := os.MkdirAll(badDir, 0755); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
badFile := filepath.Join(badDir, oid)
|
||||
if err := os.Rename(path, badFile); err != nil {
|
||||
return false, err
|
||||
}
|
||||
Print(" moved to %s", badFile)
|
||||
}
|
||||
}
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func getPointersFromRef(ref string) (map[string]string, error) {
|
||||
// The LFS scanner methods return unexported *lfs.wrappedPointer objects.
|
||||
// All we care about is the pointer OID and file name
|
||||
pointerIndex := make(map[string]string)
|
||||
|
||||
var multiErr error
|
||||
gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
|
||||
if err != nil {
|
||||
if multiErr != nil {
|
||||
multiErr = fmt.Errorf("%v\n%v", multiErr, err)
|
||||
} else {
|
||||
multiErr = err
|
||||
}
|
||||
return
|
||||
}
|
||||
pointerIndex[p.Oid] = p.Name
|
||||
})
|
||||
|
||||
defer gitscanner.Close()
|
||||
if err := gitscanner.ScanRefWithDeleted(ref, nil); err != nil {
|
||||
return pointerIndex, err
|
||||
}
|
||||
|
||||
if err := gitscanner.ScanIndex("HEAD", nil); err != nil {
|
||||
return pointerIndex, err
|
||||
}
|
||||
|
||||
return pointerIndex, multiErr
|
||||
}
|
||||
|
||||
// TODO(zeroshirts): 'git fsck' reports status (percentage, current#/total) as
|
||||
// it checks... we should do the same, as we are rehashing potentially gigs and
|
||||
// gigs of content.
|
||||
@ -118,15 +25,91 @@ func getPointersFromRef(ref string) (map[string]string, error) {
|
||||
// chain a lfs-fsck, but I don't think it does.
|
||||
func fsckCommand(cmd *cobra.Command, args []string) {
|
||||
lfs.InstallHooks(false)
|
||||
requireInRepo()
|
||||
|
||||
ok, err := doFsck()
|
||||
ref, err := git.CurrentRef()
|
||||
if err != nil {
|
||||
Panic(err, "Error checking Git LFS files")
|
||||
ExitWithError(err)
|
||||
}
|
||||
|
||||
if ok {
|
||||
Print("Git LFS fsck OK")
|
||||
var corruptOids []string
|
||||
gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
|
||||
if err == nil {
|
||||
var pointerOk bool
|
||||
pointerOk, err = fsckPointer(p.Name, p.Oid)
|
||||
if !pointerOk {
|
||||
corruptOids = append(corruptOids, p.Oid)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
Panic(err, "Error checking Git LFS files")
|
||||
}
|
||||
})
|
||||
|
||||
if err := gitscanner.ScanRefWithDeleted(ref.Sha, nil); err != nil {
|
||||
ExitWithError(err)
|
||||
}
|
||||
|
||||
if err := gitscanner.ScanIndex("HEAD", nil); err != nil {
|
||||
ExitWithError(err)
|
||||
}
|
||||
|
||||
gitscanner.Close()
|
||||
|
||||
if len(corruptOids) == 0 {
|
||||
Print("Git LFS fsck OK")
|
||||
return
|
||||
}
|
||||
|
||||
if fsckDryRun {
|
||||
return
|
||||
}
|
||||
|
||||
badDir := filepath.Join(config.LocalGitStorageDir, "lfs", "bad")
|
||||
Print("Moving corrupt objects to %s", badDir)
|
||||
|
||||
if err := os.MkdirAll(badDir, 0755); err != nil {
|
||||
ExitWithError(err)
|
||||
}
|
||||
|
||||
for _, oid := range corruptOids {
|
||||
badFile := filepath.Join(badDir, oid)
|
||||
if err := os.Rename(lfs.LocalMediaPathReadOnly(oid), badFile); err != nil {
|
||||
ExitWithError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func fsckPointer(name, oid string) (bool, error) {
|
||||
path := lfs.LocalMediaPathReadOnly(oid)
|
||||
|
||||
Debug("Examining %v (%v)", name, path)
|
||||
|
||||
f, err := os.Open(path)
|
||||
if pErr, pOk := err.(*os.PathError); pOk {
|
||||
Print("Object %s (%s) could not be checked: %s", name, oid, pErr.Err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
oidHash := sha256.New()
|
||||
_, err = io.Copy(oidHash, f)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
recalculatedOid := hex.EncodeToString(oidHash.Sum(nil))
|
||||
if recalculatedOid == oid {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
Print("Object %s (%s) is corrupt", name, oid)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@ -38,20 +38,14 @@ begin_test "fsck default"
|
||||
|
||||
echo "CORRUPTION" >> .git/lfs/objects/$aOid12/$aOid34/$aOid
|
||||
|
||||
moved=$(native_path "$TRASHDIR/$reponame/.git/lfs/bad/$aOid")
|
||||
moved=$(native_path "$TRASHDIR/$reponame/.git/lfs/bad")
|
||||
expected="$(printf 'Object a.dat (%s) is corrupt
|
||||
moved to %s' "$aOid" "$moved")"
|
||||
Moving corrupt objects to %s' "$aOid" "$moved")"
|
||||
[ "$expected" = "$(git lfs fsck)" ]
|
||||
|
||||
if [ -e .git/lfs/objects/$aOid12/$aOid34/$aOid ]; then
|
||||
echo "Expected a.dat to be cleared for being corrupt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$bOid" != "$(calc_oid_file .git/lfs/objects/$bOid12/$bOid34/$bOid)" ]; then
|
||||
echo "oid for b.dat does not match"
|
||||
exit 1
|
||||
fi
|
||||
[ -e ".git/lfs/bad/$aOid" ]
|
||||
[ ! -e ".git/lfs/objects/$aOid12/$aOid34/$aOid" ]
|
||||
[ "$bOid" = "$(calc_oid_file .git/lfs/objects/$bOid12/$bOid34/$bOid)" ]
|
||||
)
|
||||
end_test
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user