Add CLI progress bars to the packer tool

Since packing can be quite slow, it is nice to display progress to the
caller. We use the excellent github.com/vbauerster/mpb library to do so,
and add a bit of colour with github.com/logrusru/aurora.

Finally, augment the inspector and packer with a summary printer that
displays the overall file size/count, and compression ratio for each
compression type.
This commit is contained in:
Laurence Withers 2020-02-15 11:19:17 +00:00
parent 984639d475
commit 6f532296ef
6 changed files with 273 additions and 12 deletions

View File

@ -6,8 +6,10 @@ require (
github.com/andybalholm/brotli v1.0.0
github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2
github.com/kr/pretty v0.1.0 // indirect
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381
github.com/spf13/cobra v0.0.5
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1
github.com/vbauerster/mpb/v4 v4.11.2
golang.org/x/sys v0.0.0-20200116001909-b77594299b42
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/yaml.v2 v2.2.2
src.lwithers.me.uk/go/htpack v1.1.5

View File

@ -1,4 +1,8 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@ -22,6 +26,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@ -38,12 +44,24 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/vbauerster/mpb/v4 v4.11.2 h1:ynkUoKzi65DZ1UsQPx7sgi/KN6G9f7br+Us2nKm35AM=
github.com/vbauerster/mpb/v4 v4.11.2/go.mod h1:jIuIRCltGJUnm6DCyPVkwjlLUk4nHTH+m4eD14CdFF0=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4=
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 h1:gZpLHxUX5BdYLA08Lj4YCJNN/jk7KtquiArPoeX0WvA=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@ -32,7 +32,6 @@ var inspectCmd = &cobra.Command{
// Inspect a packfile.
// TODO: verify etag; verify integrity of compressed data.
// TODO: skip Gzip/Brotli if not present; print ratio.
func Inspect(filename string) error {
f, err := os.Open(filename)
if err != nil {
@ -65,10 +64,46 @@ func Inspect(filename string) error {
printSize(info.Brotli.Length), info.Brotli.Offset)
}
}
inspectSummary(dir)
}
return err
}
func inspectSummary(dir *packed.Directory) {
var (
n, ngzip, nbrotli int
s, sgzip, sbrotli uint64
)
for _, f := range dir.Files {
n++
s += f.Uncompressed.Length
if f.Gzip != nil {
ngzip++
sgzip += f.Gzip.Length
}
if f.Brotli != nil {
nbrotli++
sbrotli += f.Brotli.Length
}
}
fmt.Printf("Uncompressed:\n\tFiles: %d\n\tSize: %s\n",
n, printSize(s))
if ngzip > 0 {
fmt.Printf("gzip compressed:\n\tFiles: %d (%.1f%% of total)\n"+
"\tSize: %s\n\tRatio: %.1f%%\n",
ngzip, 100*float64(ngzip)/float64(n),
printSize(sgzip), 100*float64(sgzip)/float64(s))
}
if nbrotli > 0 {
fmt.Printf("brotli compressed:\n\tFiles: %d (%.1f%% of total)\n"+
"\tSize: %s\n\tRatio: %.1f%%\n",
nbrotli, 100*float64(nbrotli)/float64(n),
printSize(sbrotli), 100*float64(sbrotli)/float64(s))
}
}
func printSize(size uint64) string {
switch {
case size < 1<<10:

View File

@ -10,6 +10,7 @@ import (
"github.com/spf13/cobra"
yaml "gopkg.in/yaml.v2"
"src.lwithers.me.uk/go/htpack/cmd/htpacker/packer"
"src.lwithers.me.uk/go/htpack/packed"
)
var packCmd = &cobra.Command{
@ -89,7 +90,8 @@ func PackFiles(c *cobra.Command, args []string, out string) error {
if err != nil {
return err
}
return packer.Pack(ftp, out)
return doPack(ftp, out)
}
func PackSpec(c *cobra.Command, spec, out string) error {
@ -103,5 +105,26 @@ func PackSpec(c *cobra.Command, spec, out string) error {
return fmt.Errorf("parsing YAML spec %s: %v", spec, err)
}
return packer.Pack(ftp, out)
return doPack(ftp, out)
}
func doPack(ftp packer.FilesToPack, out string) error {
prog := mpbProgress(ftp)
err := packer.Pack2(ftp, out, prog)
prog.Complete()
if err == nil {
fin, err := os.Open(out)
if err != nil {
return err
}
defer fin.Close()
_, dir, err := packed.Load(fin)
if err != nil {
return err
}
inspectSummary(dir)
}
return err
}

View File

@ -0,0 +1,144 @@
package main
import (
"github.com/logrusorgru/aurora"
"github.com/vbauerster/mpb/v4"
"github.com/vbauerster/mpb/v4/decor"
"src.lwithers.me.uk/go/htpack/cmd/htpacker/packer"
)
const mpbBarStyle = "[██░]"
// mpbProgress returns a new progress object that keeps the user informed via
// the visual mpb library.
func mpbProgress(ftp packer.FilesToPack) *mpbProg {
mp := new(mpbProg)
mp.un.max = len(ftp)
for _, f := range ftp {
if !f.DisableCompression && !f.DisableGzip {
mp.gzip.max++
}
if !f.DisableCompression && !f.DisableBrotli {
mp.brotli.max++
}
}
mp.p = mpb.New()
mp.un.bar = mp.p.AddBar(int64(mp.un.max),
mpb.PrependDecorators(barName("uncompressed")),
mpb.BarStyle(mpbBarStyle),
mpb.AppendDecorators(&mp.un))
if mp.gzip.max > 0 {
mp.gzip.bar = mp.p.AddBar(int64(mp.gzip.max),
mpb.PrependDecorators(barName("gzip")),
mpb.BarStyle(mpbBarStyle),
mpb.AppendDecorators(&mp.gzip))
}
if mp.brotli.max > 0 {
mp.brotli.bar = mp.p.AddBar(int64(mp.brotli.max),
mpb.PrependDecorators(barName("brotli")),
mpb.BarStyle(mpbBarStyle),
mpb.AppendDecorators(&mp.brotli))
}
return mp
}
func barName(n string) decor.Decorator {
return decor.Name(aurora.Magenta(n).String(), decor.WCSyncWidth)
}
// mpbProg is the mpb progress tracker. It has one bar for each type of
// compression, and its methods simply dispatch onto the type-specific
// bars.
type mpbProg struct {
un, gzip, brotli mpbProg1
p *mpb.Progress
}
func (mp *mpbProg) Count(_ int) {
}
func (mp *mpbProg) Begin(filename, compression string) {
switch compression {
case "uncompressed":
mp.un.Begin(filename)
case "gzip":
mp.gzip.Begin(filename)
case "brotli":
mp.brotli.Begin(filename)
default:
return
}
}
func (mp *mpbProg) End(filename, compression string) {
switch compression {
case "uncompressed":
mp.un.End(filename)
case "gzip":
mp.gzip.End(filename)
case "brotli":
mp.brotli.End(filename)
default:
return
}
}
func (mp *mpbProg) Complete() {
mp.un.Complete()
mp.gzip.Complete()
mp.brotli.Complete()
mp.p.Wait()
}
// mpbProg1 is a type-specific progress bar. In addition to holding state and
// methods for updating the bar, it also implements decor.Decor.
type mpbProg1 struct {
max int // number of items we expect
done int // number of items completed
cur []string // list of currently-packing filenames
bar *mpb.Bar
// embedding this type lets us implement decor.Decor
decor.WC
}
func (mp1 *mpbProg1) Decor(stat *decor.Statistics) string {
if stat.Completed {
return ""
}
switch len(mp1.cur) {
case 0:
return aurora.Gray(8, "(idle)").String()
case 1:
return aurora.Blue(mp1.cur[0]).String()
default:
return aurora.Sprintf(aurora.Green("%s + %d more"), aurora.Blue(mp1.cur[0]), len(mp1.cur)-1)
}
}
func (mp1 *mpbProg1) Begin(filename string) {
mp1.cur = append(mp1.cur, filename)
}
func (mp1 *mpbProg1) End(filename string) {
for i, v := range mp1.cur {
if v == filename {
mp1.cur[i] = mp1.cur[len(mp1.cur)-1]
mp1.cur = mp1.cur[:len(mp1.cur)-1]
break
}
}
mp1.done++
if mp1.bar != nil {
mp1.bar.SetCurrent(int64(mp1.done))
}
}
func (mp1 *mpbProg1) Complete() {
if mp1.bar != nil {
mp1.bar.SetTotal(int64(mp1.max), true)
}
}

View File

@ -45,6 +45,24 @@ type FileToPack struct {
DisableBrotli bool `yaml:"disable_brotli"`
}
// Progress is a callback object which reports packing progress.
type Progress interface {
// Count reports the number of items that have begun processing.
Count(n int)
// Begin denotes the processing of an input file.
Begin(filename, compression string)
// End denotes the completion of input file processing.
End(filename, compression string)
}
type ignoreProgress int
func (ignoreProgress) Count(_ int) {}
func (ignoreProgress) Begin(_, _ string) {}
func (ignoreProgress) End(_, _ string) {}
const (
// minCompressionSaving means we'll only use the compressed version of
// the file if it's at least this many bytes smaller than the original.
@ -68,8 +86,18 @@ const (
sendfileLimit = 0x7FFFF000
)
// Pack a file.
// Pack a file. Use Pack2 for progress reporting.
func Pack(filesToPack FilesToPack, outputFilename string) error {
return Pack2(filesToPack, outputFilename, nil)
}
// Pack2 will pack a file, with progress reporting. The progress interface may
// be nil.
func Pack2(filesToPack FilesToPack, outputFilename string, progress Progress) error {
if progress == nil {
progress = ignoreProgress(0)
}
finalFname, w, err := writefile.New(outputFilename)
if err != nil {
return err
@ -79,7 +107,8 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
// we use this little structure to serialise file writes below, and
// it has a couple of convenience methods for common operations
packer := packer{
w: w,
w: w,
progress: progress,
}
// write initial header (will rewrite offset/length when known)
@ -124,6 +153,7 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
Files: make(map[string]*packed.File),
}
var count int
PackingLoop:
for path, fileToPack := range filesToPack {
select {
@ -132,6 +162,8 @@ PackingLoop:
break PackingLoop
default:
packer.packFile(path, fileToPack)
count++
progress.Count(count)
}
}
@ -199,12 +231,13 @@ func compressionWorthwhile(data []byte, compressed os.FileInfo) bool {
// of compression. Unexported methods assume they are called in a context where
// the lock is not needed or already taken; exported methods take the lock.
type packer struct {
w *os.File
lock sync.Mutex
cpus chan struct{}
errors chan error
aborted chan struct{}
dir *packed.Directory
w *os.File
lock sync.Mutex
cpus chan struct{}
errors chan error
aborted chan struct{}
dir *packed.Directory
progress Progress
}
// pad will move the file write pointer to the next padding boundary. It is not
@ -312,11 +345,15 @@ func (p *packer) packFile(path string, fileToPack FileToPack) {
// list of operations on this input file that we'll carry out asynchronously
ops := []func() error{
func() error {
p.progress.Begin(fileToPack.Filename, "uncompressed")
defer p.progress.End(fileToPack.Filename, "uncompressed")
return p.Uncompressed(fileToPack.Filename, info)
},
}
if !fileToPack.DisableCompression && !fileToPack.DisableGzip {
ops = append(ops, func() error {
p.progress.Begin(fileToPack.Filename, "gzip")
defer p.progress.End(fileToPack.Filename, "gzip")
if err := p.Gzip(data, info); err != nil {
return fmt.Errorf("gzip compression of %s "+
"failed: %v", fileToPack.Filename, err)
@ -326,6 +363,8 @@ func (p *packer) packFile(path string, fileToPack FileToPack) {
}
if !fileToPack.DisableCompression && !fileToPack.DisableBrotli {
ops = append(ops, func() error {
p.progress.Begin(fileToPack.Filename, "brotli")
defer p.progress.End(fileToPack.Filename, "brotli")
if err := p.Brotli(data, info); err != nil {
return fmt.Errorf("brotli compression of %s "+
"failed: %v", fileToPack.Filename, err)