From 0569c1753cd4afde982e19acbd222a3747e43d93 Mon Sep 17 00:00:00 2001 From: risk danger olson Date: Tue, 29 Nov 2016 13:56:47 -0700 Subject: [PATCH] rewrite fsck command to use gitscanner api --- commands/command_fsck.go | 177 ++++++++++++++++++--------------------- test/test-fsck.sh | 16 ++-- 2 files changed, 85 insertions(+), 108 deletions(-) diff --git a/commands/command_fsck.go b/commands/command_fsck.go index 1f08c130..ee9e4b5a 100644 --- a/commands/command_fsck.go +++ b/commands/command_fsck.go @@ -3,7 +3,6 @@ package commands import ( "crypto/sha256" "encoding/hex" - "fmt" "io" "os" "path/filepath" @@ -18,98 +17,6 @@ var ( fsckDryRun bool ) -func doFsck() (bool, error) { - requireInRepo() - - ref, err := git.CurrentRef() - if err != nil { - return false, err - } - - // The LFS scanner methods return unexported *lfs.wrappedPointer objects. - // All we care about is the pointer OID and file name - pointerIndex, err := getPointersFromRef(ref.Sha) - if err != nil { - return false, err - } - - ok := true - - for oid, name := range pointerIndex { - path := lfs.LocalMediaPathReadOnly(oid) - - Debug("Examining %v (%v)", name, path) - - f, err := os.Open(path) - if pErr, pOk := err.(*os.PathError); pOk { - Print("Object %s (%s) could not be checked: %s", name, oid, pErr.Err) - ok = false - continue - } - if err != nil { - return false, err - } - - oidHash := sha256.New() - _, err = io.Copy(oidHash, f) - f.Close() - if err != nil { - return false, err - } - - recalculatedOid := hex.EncodeToString(oidHash.Sum(nil)) - if recalculatedOid != oid { - ok = false - Print("Object %s (%s) is corrupt", name, oid) - if fsckDryRun { - continue - } - - badDir := filepath.Join(config.LocalGitStorageDir, "lfs", "bad") - if err := os.MkdirAll(badDir, 0755); err != nil { - return false, err - } - - badFile := filepath.Join(badDir, oid) - if err := os.Rename(path, badFile); err != nil { - return false, err - } - Print(" moved to %s", badFile) - } - } - return ok, nil -} - -func getPointersFromRef(ref string) (map[string]string, error) { - // The LFS scanner methods return unexported *lfs.wrappedPointer objects. - // All we care about is the pointer OID and file name - pointerIndex := make(map[string]string) - - var multiErr error - gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { - if err != nil { - if multiErr != nil { - multiErr = fmt.Errorf("%v\n%v", multiErr, err) - } else { - multiErr = err - } - return - } - pointerIndex[p.Oid] = p.Name - }) - - defer gitscanner.Close() - if err := gitscanner.ScanRefWithDeleted(ref, nil); err != nil { - return pointerIndex, err - } - - if err := gitscanner.ScanIndex("HEAD", nil); err != nil { - return pointerIndex, err - } - - return pointerIndex, multiErr -} - // TODO(zeroshirts): 'git fsck' reports status (percentage, current#/total) as // it checks... we should do the same, as we are rehashing potentially gigs and // gigs of content. @@ -118,15 +25,91 @@ func getPointersFromRef(ref string) (map[string]string, error) { // chain a lfs-fsck, but I don't think it does. func fsckCommand(cmd *cobra.Command, args []string) { lfs.InstallHooks(false) + requireInRepo() - ok, err := doFsck() + ref, err := git.CurrentRef() if err != nil { - Panic(err, "Error checking Git LFS files") + ExitWithError(err) } - if ok { - Print("Git LFS fsck OK") + var corruptOids []string + gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { + if err == nil { + var pointerOk bool + pointerOk, err = fsckPointer(p.Name, p.Oid) + if !pointerOk { + corruptOids = append(corruptOids, p.Oid) + } + } + + if err != nil { + Panic(err, "Error checking Git LFS files") + } + }) + + if err := gitscanner.ScanRefWithDeleted(ref.Sha, nil); err != nil { + ExitWithError(err) } + + if err := gitscanner.ScanIndex("HEAD", nil); err != nil { + ExitWithError(err) + } + + gitscanner.Close() + + if len(corruptOids) == 0 { + Print("Git LFS fsck OK") + return + } + + if fsckDryRun { + return + } + + badDir := filepath.Join(config.LocalGitStorageDir, "lfs", "bad") + Print("Moving corrupt objects to %s", badDir) + + if err := os.MkdirAll(badDir, 0755); err != nil { + ExitWithError(err) + } + + for _, oid := range corruptOids { + badFile := filepath.Join(badDir, oid) + if err := os.Rename(lfs.LocalMediaPathReadOnly(oid), badFile); err != nil { + ExitWithError(err) + } + } +} + +func fsckPointer(name, oid string) (bool, error) { + path := lfs.LocalMediaPathReadOnly(oid) + + Debug("Examining %v (%v)", name, path) + + f, err := os.Open(path) + if pErr, pOk := err.(*os.PathError); pOk { + Print("Object %s (%s) could not be checked: %s", name, oid, pErr.Err) + return false, nil + } + + if err != nil { + return false, err + } + + oidHash := sha256.New() + _, err = io.Copy(oidHash, f) + f.Close() + if err != nil { + return false, err + } + + recalculatedOid := hex.EncodeToString(oidHash.Sum(nil)) + if recalculatedOid == oid { + return true, nil + } + + Print("Object %s (%s) is corrupt", name, oid) + return false, nil } func init() { diff --git a/test/test-fsck.sh b/test/test-fsck.sh index 67572e54..6d6ccfa8 100755 --- a/test/test-fsck.sh +++ b/test/test-fsck.sh @@ -38,20 +38,14 @@ begin_test "fsck default" echo "CORRUPTION" >> .git/lfs/objects/$aOid12/$aOid34/$aOid - moved=$(native_path "$TRASHDIR/$reponame/.git/lfs/bad/$aOid") + moved=$(native_path "$TRASHDIR/$reponame/.git/lfs/bad") expected="$(printf 'Object a.dat (%s) is corrupt - moved to %s' "$aOid" "$moved")" +Moving corrupt objects to %s' "$aOid" "$moved")" [ "$expected" = "$(git lfs fsck)" ] - if [ -e .git/lfs/objects/$aOid12/$aOid34/$aOid ]; then - echo "Expected a.dat to be cleared for being corrupt" - exit 1 - fi - - if [ "$bOid" != "$(calc_oid_file .git/lfs/objects/$bOid12/$bOid34/$bOid)" ]; then - echo "oid for b.dat does not match" - exit 1 - fi + [ -e ".git/lfs/bad/$aOid" ] + [ ! -e ".git/lfs/objects/$aOid12/$aOid34/$aOid" ] + [ "$bOid" = "$(calc_oid_file .git/lfs/objects/$bOid12/$bOid34/$bOid)" ] ) end_test