diff --git a/cmd/htpacker/go.mod b/cmd/htpacker/go.mod index 1ca12cf..6c5be0a 100644 --- a/cmd/htpacker/go.mod +++ b/cmd/htpacker/go.mod @@ -6,8 +6,10 @@ require ( github.com/andybalholm/brotli v1.0.0 github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2 github.com/kr/pretty v0.1.0 // indirect + github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 github.com/spf13/cobra v0.0.5 - golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 + github.com/vbauerster/mpb/v4 v4.11.2 + golang.org/x/sys v0.0.0-20200116001909-b77594299b42 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v2 v2.2.2 src.lwithers.me.uk/go/htpack v1.1.5 diff --git a/cmd/htpacker/go.sum b/cmd/htpacker/go.sum index c13ee62..d605785 100644 --- a/cmd/htpacker/go.sum +++ b/cmd/htpacker/go.sum @@ -1,4 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -22,6 +26,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= +github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -38,12 +44,24 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/vbauerster/mpb/v4 v4.11.2 h1:ynkUoKzi65DZ1UsQPx7sgi/KN6G9f7br+Us2nKm35AM= +github.com/vbauerster/mpb/v4 v4.11.2/go.mod h1:jIuIRCltGJUnm6DCyPVkwjlLUk4nHTH+m4eD14CdFF0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= +golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 h1:gZpLHxUX5BdYLA08Lj4YCJNN/jk7KtquiArPoeX0WvA= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/cmd/htpacker/inspector.go b/cmd/htpacker/inspector.go index 6480ee8..9d3dab8 100644 --- a/cmd/htpacker/inspector.go +++ b/cmd/htpacker/inspector.go @@ -32,7 +32,6 @@ var inspectCmd = &cobra.Command{ // Inspect a packfile. // TODO: verify etag; verify integrity of compressed data. -// TODO: skip Gzip/Brotli if not present; print ratio. func Inspect(filename string) error { f, err := os.Open(filename) if err != nil { @@ -65,10 +64,46 @@ func Inspect(filename string) error { printSize(info.Brotli.Length), info.Brotli.Offset) } } + inspectSummary(dir) } return err } +func inspectSummary(dir *packed.Directory) { + var ( + n, ngzip, nbrotli int + s, sgzip, sbrotli uint64 + ) + + for _, f := range dir.Files { + n++ + s += f.Uncompressed.Length + if f.Gzip != nil { + ngzip++ + sgzip += f.Gzip.Length + } + if f.Brotli != nil { + nbrotli++ + sbrotli += f.Brotli.Length + } + } + + fmt.Printf("Uncompressed:\n\tFiles: %d\n\tSize: %s\n", + n, printSize(s)) + if ngzip > 0 { + fmt.Printf("gzip compressed:\n\tFiles: %d (%.1f%% of total)\n"+ + "\tSize: %s\n\tRatio: %.1f%%\n", + ngzip, 100*float64(ngzip)/float64(n), + printSize(sgzip), 100*float64(sgzip)/float64(s)) + } + if nbrotli > 0 { + fmt.Printf("brotli compressed:\n\tFiles: %d (%.1f%% of total)\n"+ + "\tSize: %s\n\tRatio: %.1f%%\n", + nbrotli, 100*float64(nbrotli)/float64(n), + printSize(sbrotli), 100*float64(sbrotli)/float64(s)) + } +} + func printSize(size uint64) string { switch { case size < 1<<10: diff --git a/cmd/htpacker/pack.go b/cmd/htpacker/pack.go index debe8cb..f584616 100644 --- a/cmd/htpacker/pack.go +++ b/cmd/htpacker/pack.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" yaml "gopkg.in/yaml.v2" "src.lwithers.me.uk/go/htpack/cmd/htpacker/packer" + "src.lwithers.me.uk/go/htpack/packed" ) var packCmd = &cobra.Command{ @@ -89,7 +90,8 @@ func PackFiles(c *cobra.Command, args []string, out string) error { if err != nil { return err } - return packer.Pack(ftp, out) + + return doPack(ftp, out) } func PackSpec(c *cobra.Command, spec, out string) error { @@ -103,5 +105,26 @@ func PackSpec(c *cobra.Command, spec, out string) error { return fmt.Errorf("parsing YAML spec %s: %v", spec, err) } - return packer.Pack(ftp, out) + return doPack(ftp, out) +} + +func doPack(ftp packer.FilesToPack, out string) error { + prog := mpbProgress(ftp) + err := packer.Pack2(ftp, out, prog) + prog.Complete() + + if err == nil { + fin, err := os.Open(out) + if err != nil { + return err + } + defer fin.Close() + + _, dir, err := packed.Load(fin) + if err != nil { + return err + } + inspectSummary(dir) + } + return err } diff --git a/cmd/htpacker/pack_progress.go b/cmd/htpacker/pack_progress.go new file mode 100644 index 0000000..9f7ce88 --- /dev/null +++ b/cmd/htpacker/pack_progress.go @@ -0,0 +1,144 @@ +package main + +import ( + "github.com/logrusorgru/aurora" + "github.com/vbauerster/mpb/v4" + "github.com/vbauerster/mpb/v4/decor" + "src.lwithers.me.uk/go/htpack/cmd/htpacker/packer" +) + +const mpbBarStyle = "[██░]" + +// mpbProgress returns a new progress object that keeps the user informed via +// the visual mpb library. +func mpbProgress(ftp packer.FilesToPack) *mpbProg { + mp := new(mpbProg) + mp.un.max = len(ftp) + + for _, f := range ftp { + if !f.DisableCompression && !f.DisableGzip { + mp.gzip.max++ + } + if !f.DisableCompression && !f.DisableBrotli { + mp.brotli.max++ + } + } + + mp.p = mpb.New() + mp.un.bar = mp.p.AddBar(int64(mp.un.max), + mpb.PrependDecorators(barName("uncompressed")), + mpb.BarStyle(mpbBarStyle), + mpb.AppendDecorators(&mp.un)) + if mp.gzip.max > 0 { + mp.gzip.bar = mp.p.AddBar(int64(mp.gzip.max), + mpb.PrependDecorators(barName("gzip")), + mpb.BarStyle(mpbBarStyle), + mpb.AppendDecorators(&mp.gzip)) + } + if mp.brotli.max > 0 { + mp.brotli.bar = mp.p.AddBar(int64(mp.brotli.max), + mpb.PrependDecorators(barName("brotli")), + mpb.BarStyle(mpbBarStyle), + mpb.AppendDecorators(&mp.brotli)) + } + + return mp +} + +func barName(n string) decor.Decorator { + return decor.Name(aurora.Magenta(n).String(), decor.WCSyncWidth) +} + +// mpbProg is the mpb progress tracker. It has one bar for each type of +// compression, and its methods simply dispatch onto the type-specific +// bars. +type mpbProg struct { + un, gzip, brotli mpbProg1 + p *mpb.Progress +} + +func (mp *mpbProg) Count(_ int) { +} + +func (mp *mpbProg) Begin(filename, compression string) { + switch compression { + case "uncompressed": + mp.un.Begin(filename) + case "gzip": + mp.gzip.Begin(filename) + case "brotli": + mp.brotli.Begin(filename) + default: + return + } +} + +func (mp *mpbProg) End(filename, compression string) { + switch compression { + case "uncompressed": + mp.un.End(filename) + case "gzip": + mp.gzip.End(filename) + case "brotli": + mp.brotli.End(filename) + default: + return + } +} + +func (mp *mpbProg) Complete() { + mp.un.Complete() + mp.gzip.Complete() + mp.brotli.Complete() + mp.p.Wait() +} + +// mpbProg1 is a type-specific progress bar. In addition to holding state and +// methods for updating the bar, it also implements decor.Decor. +type mpbProg1 struct { + max int // number of items we expect + done int // number of items completed + cur []string // list of currently-packing filenames + bar *mpb.Bar + + // embedding this type lets us implement decor.Decor + decor.WC +} + +func (mp1 *mpbProg1) Decor(stat *decor.Statistics) string { + if stat.Completed { + return "" + } + switch len(mp1.cur) { + case 0: + return aurora.Gray(8, "(idle)").String() + case 1: + return aurora.Blue(mp1.cur[0]).String() + default: + return aurora.Sprintf(aurora.Green("%s + %d more"), aurora.Blue(mp1.cur[0]), len(mp1.cur)-1) + } +} + +func (mp1 *mpbProg1) Begin(filename string) { + mp1.cur = append(mp1.cur, filename) +} + +func (mp1 *mpbProg1) End(filename string) { + for i, v := range mp1.cur { + if v == filename { + mp1.cur[i] = mp1.cur[len(mp1.cur)-1] + mp1.cur = mp1.cur[:len(mp1.cur)-1] + break + } + } + mp1.done++ + if mp1.bar != nil { + mp1.bar.SetCurrent(int64(mp1.done)) + } +} + +func (mp1 *mpbProg1) Complete() { + if mp1.bar != nil { + mp1.bar.SetTotal(int64(mp1.max), true) + } +} diff --git a/cmd/htpacker/packer/packer.go b/cmd/htpacker/packer/packer.go index f3baf5e..00cc31e 100644 --- a/cmd/htpacker/packer/packer.go +++ b/cmd/htpacker/packer/packer.go @@ -45,6 +45,24 @@ type FileToPack struct { DisableBrotli bool `yaml:"disable_brotli"` } +// Progress is a callback object which reports packing progress. +type Progress interface { + // Count reports the number of items that have begun processing. + Count(n int) + + // Begin denotes the processing of an input file. + Begin(filename, compression string) + + // End denotes the completion of input file processing. + End(filename, compression string) +} + +type ignoreProgress int + +func (ignoreProgress) Count(_ int) {} +func (ignoreProgress) Begin(_, _ string) {} +func (ignoreProgress) End(_, _ string) {} + const ( // minCompressionSaving means we'll only use the compressed version of // the file if it's at least this many bytes smaller than the original. @@ -68,8 +86,18 @@ const ( sendfileLimit = 0x7FFFF000 ) -// Pack a file. +// Pack a file. Use Pack2 for progress reporting. func Pack(filesToPack FilesToPack, outputFilename string) error { + return Pack2(filesToPack, outputFilename, nil) +} + +// Pack2 will pack a file, with progress reporting. The progress interface may +// be nil. +func Pack2(filesToPack FilesToPack, outputFilename string, progress Progress) error { + if progress == nil { + progress = ignoreProgress(0) + } + finalFname, w, err := writefile.New(outputFilename) if err != nil { return err @@ -79,7 +107,8 @@ func Pack(filesToPack FilesToPack, outputFilename string) error { // we use this little structure to serialise file writes below, and // it has a couple of convenience methods for common operations packer := packer{ - w: w, + w: w, + progress: progress, } // write initial header (will rewrite offset/length when known) @@ -124,6 +153,7 @@ func Pack(filesToPack FilesToPack, outputFilename string) error { Files: make(map[string]*packed.File), } + var count int PackingLoop: for path, fileToPack := range filesToPack { select { @@ -132,6 +162,8 @@ PackingLoop: break PackingLoop default: packer.packFile(path, fileToPack) + count++ + progress.Count(count) } } @@ -199,12 +231,13 @@ func compressionWorthwhile(data []byte, compressed os.FileInfo) bool { // of compression. Unexported methods assume they are called in a context where // the lock is not needed or already taken; exported methods take the lock. type packer struct { - w *os.File - lock sync.Mutex - cpus chan struct{} - errors chan error - aborted chan struct{} - dir *packed.Directory + w *os.File + lock sync.Mutex + cpus chan struct{} + errors chan error + aborted chan struct{} + dir *packed.Directory + progress Progress } // pad will move the file write pointer to the next padding boundary. It is not @@ -312,11 +345,15 @@ func (p *packer) packFile(path string, fileToPack FileToPack) { // list of operations on this input file that we'll carry out asynchronously ops := []func() error{ func() error { + p.progress.Begin(fileToPack.Filename, "uncompressed") + defer p.progress.End(fileToPack.Filename, "uncompressed") return p.Uncompressed(fileToPack.Filename, info) }, } if !fileToPack.DisableCompression && !fileToPack.DisableGzip { ops = append(ops, func() error { + p.progress.Begin(fileToPack.Filename, "gzip") + defer p.progress.End(fileToPack.Filename, "gzip") if err := p.Gzip(data, info); err != nil { return fmt.Errorf("gzip compression of %s "+ "failed: %v", fileToPack.Filename, err) @@ -326,6 +363,8 @@ func (p *packer) packFile(path string, fileToPack FileToPack) { } if !fileToPack.DisableCompression && !fileToPack.DisableBrotli { ops = append(ops, func() error { + p.progress.Begin(fileToPack.Filename, "brotli") + defer p.progress.End(fileToPack.Filename, "brotli") if err := p.Brotli(data, info); err != nil { return fmt.Errorf("brotli compression of %s "+ "failed: %v", fileToPack.Filename, err)