2017-06-09 23:53:13 +00:00
|
|
|
package commands
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"sort"
|
|
|
|
"strings"
|
|
|
|
|
2017-06-13 21:59:36 +00:00
|
|
|
"github.com/git-lfs/git-lfs/errors"
|
2017-06-21 16:54:55 +00:00
|
|
|
"github.com/git-lfs/git-lfs/git/githistory"
|
2017-11-22 22:07:24 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tasklog"
|
2017-06-09 23:53:13 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tools"
|
2017-06-13 21:59:36 +00:00
|
|
|
"github.com/git-lfs/git-lfs/tools/humanize"
|
2018-07-05 16:49:10 +00:00
|
|
|
"github.com/git-lfs/gitobj"
|
2017-06-09 23:53:13 +00:00
|
|
|
"github.com/spf13/cobra"
|
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
|
|
|
// migrateInfoTopN is a flag given to the git-lfs-migrate(1) subcommand
|
|
|
|
// 'info' which specifies how many info entries to show by default.
|
|
|
|
migrateInfoTopN int
|
2017-06-13 21:59:36 +00:00
|
|
|
|
|
|
|
// migrateInfoAboveFmt is a flag given to the git-lfs-migrate(1)
|
|
|
|
// subcommand 'info' specifying a human-readable string threshold of
|
|
|
|
// filesize before entries are counted.
|
|
|
|
migrateInfoAboveFmt string
|
|
|
|
// migrateInfoAbove is the number of bytes parsed from the above
|
|
|
|
// migrateInfoAboveFmt flag.
|
|
|
|
migrateInfoAbove uint64
|
2017-06-15 21:08:15 +00:00
|
|
|
|
|
|
|
// migrateInfoUnitFmt is a flag given to the git-lfs-migrate(1)
|
|
|
|
// subcommand 'info' specifying a human-readable string of units with
|
|
|
|
// which to display the number of bytes.
|
|
|
|
migrateInfoUnitFmt string
|
|
|
|
// migrateInfoUnit is the number of bytes in the unit given as
|
|
|
|
// migrateInfoUnitFmt.
|
|
|
|
migrateInfoUnit uint64
|
2017-06-09 23:53:13 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func migrateInfoCommand(cmd *cobra.Command, args []string) {
|
2018-10-30 08:06:55 +00:00
|
|
|
l := tasklog.NewLogger(os.Stderr,
|
|
|
|
tasklog.ForceProgress(cfg.ForceProgress()),
|
|
|
|
)
|
2017-08-29 21:05:11 +00:00
|
|
|
|
2017-06-15 19:52:55 +00:00
|
|
|
db, err := getObjectDatabase()
|
|
|
|
if err != nil {
|
|
|
|
ExitWithError(err)
|
|
|
|
}
|
2017-09-19 20:07:37 +00:00
|
|
|
defer db.Close()
|
|
|
|
|
2017-08-29 21:05:11 +00:00
|
|
|
rewriter := getHistoryRewriter(cmd, db, l)
|
2017-06-15 19:52:55 +00:00
|
|
|
|
2017-06-13 21:59:36 +00:00
|
|
|
exts := make(map[string]*MigrateInfoEntry)
|
|
|
|
|
|
|
|
above, err := humanize.ParseBytes(migrateInfoAboveFmt)
|
|
|
|
if err != nil {
|
|
|
|
ExitWithError(errors.Wrap(err, "cannot parse --above=<n>"))
|
|
|
|
}
|
|
|
|
|
2017-06-15 21:08:15 +00:00
|
|
|
if u := cmd.Flag("unit"); u.Changed {
|
|
|
|
unit, err := humanize.ParseByteUnit(u.Value.String())
|
|
|
|
if err != nil {
|
|
|
|
ExitWithError(errors.Wrap(err, "cannot parse --unit=<unit>"))
|
|
|
|
}
|
|
|
|
|
|
|
|
migrateInfoUnit = unit
|
|
|
|
}
|
|
|
|
|
2017-06-13 21:59:36 +00:00
|
|
|
migrateInfoAbove = above
|
2017-06-09 23:53:13 +00:00
|
|
|
|
2017-08-29 21:05:11 +00:00
|
|
|
migrate(args, rewriter, l, &githistory.RewriteOptions{
|
2018-07-05 16:49:10 +00:00
|
|
|
BlobFn: func(path string, b *gitobj.Blob) (*gitobj.Blob, error) {
|
2017-06-21 16:54:55 +00:00
|
|
|
ext := fmt.Sprintf("*%s", filepath.Ext(path))
|
2017-06-13 21:59:36 +00:00
|
|
|
|
2017-06-21 16:54:55 +00:00
|
|
|
if len(ext) > 1 {
|
|
|
|
entry := exts[ext]
|
|
|
|
if entry == nil {
|
|
|
|
entry = &MigrateInfoEntry{Qualifier: ext}
|
|
|
|
}
|
2017-06-13 21:59:36 +00:00
|
|
|
|
2017-06-21 16:54:55 +00:00
|
|
|
entry.Total++
|
|
|
|
entry.BytesTotal += b.Size
|
2017-06-13 21:59:36 +00:00
|
|
|
|
2017-06-21 16:54:55 +00:00
|
|
|
if b.Size > int64(migrateInfoAbove) {
|
|
|
|
entry.TotalAbove++
|
|
|
|
entry.BytesAbove += b.Size
|
|
|
|
}
|
2017-06-13 21:59:36 +00:00
|
|
|
|
2017-06-21 16:54:55 +00:00
|
|
|
exts[ext] = entry
|
|
|
|
}
|
2017-06-09 23:53:13 +00:00
|
|
|
|
2017-06-21 16:54:55 +00:00
|
|
|
return b, nil
|
|
|
|
},
|
2017-06-09 23:53:13 +00:00
|
|
|
})
|
2017-09-22 22:28:07 +00:00
|
|
|
l.Close()
|
2017-06-09 23:53:13 +00:00
|
|
|
|
|
|
|
entries := EntriesBySize(MapToEntries(exts))
|
2017-06-26 15:24:09 +00:00
|
|
|
entries = removeEmptyEntries(entries)
|
2017-06-09 23:53:13 +00:00
|
|
|
sort.Sort(sort.Reverse(entries))
|
|
|
|
|
2017-06-10 01:09:54 +00:00
|
|
|
migrateInfoTopN = tools.ClampInt(migrateInfoTopN, len(entries), 0)
|
2017-06-09 23:53:13 +00:00
|
|
|
|
|
|
|
entries = entries[:tools.MaxInt(0, migrateInfoTopN)]
|
|
|
|
|
2017-06-15 04:33:36 +00:00
|
|
|
entries.Print(os.Stdout)
|
2017-06-09 23:53:13 +00:00
|
|
|
}
|
|
|
|
|
2017-06-13 23:24:30 +00:00
|
|
|
// MigrateInfoEntry represents a tuple of filetype to bytes and entry count
|
|
|
|
// above and below a threshold.
|
2017-06-09 23:53:13 +00:00
|
|
|
type MigrateInfoEntry struct {
|
|
|
|
// Qualifier is the filepath's extension.
|
|
|
|
Qualifier string
|
2017-06-13 21:59:36 +00:00
|
|
|
|
2017-06-13 23:24:30 +00:00
|
|
|
// BytesAbove is total size of all files above a given threshold.
|
2017-06-13 21:59:36 +00:00
|
|
|
BytesAbove int64
|
2017-06-13 23:24:30 +00:00
|
|
|
// TotalAbove is the count of all files above a given size threshold.
|
2017-06-13 21:59:36 +00:00
|
|
|
TotalAbove int64
|
2017-06-13 23:24:30 +00:00
|
|
|
// BytesTotal is the number of bytes of all files
|
2017-06-13 21:59:36 +00:00
|
|
|
BytesTotal int64
|
2017-06-13 23:24:30 +00:00
|
|
|
// Total is the count of all files.
|
|
|
|
Total int64
|
2017-06-09 23:53:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// MapToEntries creates a set of `*MigrateInfoEntry`'s for a given map of
|
|
|
|
// filepath extensions to file size in bytes.
|
2017-06-13 21:59:36 +00:00
|
|
|
func MapToEntries(exts map[string]*MigrateInfoEntry) []*MigrateInfoEntry {
|
2017-06-09 23:53:13 +00:00
|
|
|
entries := make([]*MigrateInfoEntry, 0, len(exts))
|
2017-06-13 21:59:36 +00:00
|
|
|
for _, entry := range exts {
|
|
|
|
entries = append(entries, entry)
|
2017-06-09 23:53:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return entries
|
|
|
|
}
|
|
|
|
|
2017-06-26 15:24:09 +00:00
|
|
|
// removeEmptyEntries removes `*MigrateInfoEntry`'s for which no matching file
|
|
|
|
// is above the given threshold "--above".
|
|
|
|
func removeEmptyEntries(entries []*MigrateInfoEntry) []*MigrateInfoEntry {
|
|
|
|
nz := make([]*MigrateInfoEntry, 0, len(entries))
|
|
|
|
for _, e := range entries {
|
|
|
|
if e.TotalAbove > 0 {
|
|
|
|
nz = append(nz, e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nz
|
|
|
|
}
|
|
|
|
|
2017-06-09 23:53:13 +00:00
|
|
|
// EntriesBySize is an implementation of sort.Interface that sorts a set of
|
|
|
|
// `*MigrateInfoEntry`'s
|
|
|
|
type EntriesBySize []*MigrateInfoEntry
|
|
|
|
|
|
|
|
// Len returns the total length of the set of `*MigrateInfoEntry`'s.
|
|
|
|
func (e EntriesBySize) Len() int { return len(e) }
|
|
|
|
|
|
|
|
// Less returns the whether or not the MigrateInfoEntry given at `i` takes up
|
|
|
|
// less total size than the MigrateInfoEntry given at `j`.
|
2017-06-13 21:59:36 +00:00
|
|
|
func (e EntriesBySize) Less(i, j int) bool { return e[i].BytesAbove < e[j].BytesAbove }
|
2017-06-09 23:53:13 +00:00
|
|
|
|
|
|
|
// Swap swaps the entries given at i, j.
|
|
|
|
func (e EntriesBySize) Swap(i, j int) { e[i], e[j] = e[j], e[i] }
|
|
|
|
|
|
|
|
// Print formats the `*MigrateInfoEntry`'s in the set and prints them to the
|
|
|
|
// given io.Writer, "to", returning "n" the number of bytes written, and any
|
|
|
|
// error, if one occurred.
|
|
|
|
func (e EntriesBySize) Print(to io.Writer) (int, error) {
|
2017-06-26 15:24:09 +00:00
|
|
|
if len(e) == 0 {
|
|
|
|
return 0, nil
|
|
|
|
}
|
|
|
|
|
2017-06-13 21:59:36 +00:00
|
|
|
extensions := make([]string, 0, len(e))
|
2017-06-13 23:35:23 +00:00
|
|
|
sizes := make([]string, 0, len(e))
|
2017-06-13 23:29:57 +00:00
|
|
|
stats := make([]string, 0, len(e))
|
2017-06-13 23:28:26 +00:00
|
|
|
percentages := make([]string, 0, len(e))
|
|
|
|
|
2017-06-13 21:59:36 +00:00
|
|
|
for _, entry := range e {
|
2017-06-15 21:08:15 +00:00
|
|
|
bytesAbove := uint64(entry.BytesAbove)
|
2017-06-13 21:59:36 +00:00
|
|
|
above := entry.TotalAbove
|
|
|
|
total := entry.Total
|
2017-06-13 23:28:26 +00:00
|
|
|
percentAbove := 100 * (float64(above) / float64(total))
|
2017-06-13 21:59:36 +00:00
|
|
|
|
2017-06-15 21:08:15 +00:00
|
|
|
var size string
|
|
|
|
if migrateInfoUnit > 0 {
|
|
|
|
size = humanize.FormatBytesUnit(bytesAbove, migrateInfoUnit)
|
|
|
|
} else {
|
|
|
|
size = humanize.FormatBytes(bytesAbove)
|
|
|
|
}
|
|
|
|
|
2017-06-13 23:35:23 +00:00
|
|
|
stat := fmt.Sprintf("%d/%d files(s)",
|
|
|
|
above, total)
|
2017-06-13 21:59:36 +00:00
|
|
|
|
|
|
|
percentage := fmt.Sprintf("%.0f%%", percentAbove)
|
|
|
|
|
2017-06-13 23:28:26 +00:00
|
|
|
extensions = append(extensions, entry.Qualifier)
|
2017-06-15 21:08:15 +00:00
|
|
|
sizes = append(sizes, size)
|
2017-06-13 23:29:57 +00:00
|
|
|
stats = append(stats, stat)
|
2017-06-13 21:59:36 +00:00
|
|
|
percentages = append(percentages, percentage)
|
|
|
|
}
|
2017-06-13 23:28:26 +00:00
|
|
|
|
|
|
|
extensions = tools.Ljust(extensions)
|
2017-06-13 23:35:23 +00:00
|
|
|
sizes = tools.Ljust(sizes)
|
2017-06-13 23:29:57 +00:00
|
|
|
stats = tools.Rjust(stats)
|
2017-06-13 21:59:36 +00:00
|
|
|
percentages = tools.Rjust(percentages)
|
|
|
|
|
|
|
|
output := make([]string, 0, len(e))
|
|
|
|
for i := 0; i < len(e); i++ {
|
|
|
|
extension := extensions[i]
|
2017-06-13 23:35:23 +00:00
|
|
|
size := sizes[i]
|
2017-06-13 23:29:57 +00:00
|
|
|
stat := stats[i]
|
2017-06-13 21:59:36 +00:00
|
|
|
percentage := percentages[i]
|
|
|
|
|
2017-06-13 23:35:23 +00:00
|
|
|
line := strings.Join([]string{extension, size, stat, percentage}, "\t")
|
2017-06-09 23:53:13 +00:00
|
|
|
|
|
|
|
output = append(output, line)
|
|
|
|
}
|
|
|
|
|
|
|
|
return fmt.Fprintln(to, strings.Join(output, "\n"))
|
|
|
|
}
|