Compare commits

...

4 Commits

Author SHA1 Message Date
Laurence Withers ca54fb8fbb Add doc link to README.md 2020-02-15 12:27:45 +00:00
Laurence Withers d5b4fcf0be Implement library support for Angular-style single page applications 2020-02-15 12:26:01 +00:00
Laurence Withers f08165b0f1 Update README to remove reference to external brotli command
Since we're now using github.com/andybalholm/brotli to natively pack, there's no need to refer
to an external brotli process any more.
2020-02-15 11:23:42 +00:00
Laurence Withers 6f532296ef Add CLI progress bars to the packer tool
Since packing can be quite slow, it is nice to display progress to the
caller. We use the excellent github.com/vbauerster/mpb library to do so,
and add a bit of colour with github.com/logrusru/aurora.

Finally, augment the inspector and packer with a summary printer that
displays the overall file size/count, and compression ratio for each
compression type.
2020-02-15 11:22:24 +00:00
8 changed files with 336 additions and 15 deletions

View File

@ -1,5 +1,7 @@
# HTTP resource pack server
[![GoDoc](https://img.shields.io/static/v1?label=godoc&message=reference&color=blue)](https://pkg.go.dev/src.lwithers.me.uk/go/htpack)
A common scenario is that you have a set of static resources that you want to
serve up quickly via HTTP (for example: stylesheets, WASM).
@ -7,7 +9,7 @@ This package provides a `net/http`-compatible `http.Handler` to do so, with
support for:
- compression
- gzip
- brotli, if you have the external compression binary available at pack time
- brotli
- does not yet support Transfer-Encoding, only Accept-Encoding/Content-Encoding
- etags
- ranges
@ -28,3 +30,35 @@ else will be ignored.
The interaction between range handling and compression also seems a little
ill-defined; as we have pre-compressed data, however, we can consistently
serve the exact same byte data for compressed files.
## Angular-style single-page application handling
If you wish to support an angular.js-style single page application, in which
a Javascript application uses the browser's history API to create a set of
virtual paths ("routes"), it is necessary to somehow intercept HTTP 404 errors
being returned from the handler and instead return an HTTP 200 with an HTML
document.
This can be achieved with a number of methods.
If you have an nginx instance reverse proxying in front of `htpack`, then you
can use a couple of extra directives, for example:
# prevent page loaded at "http://server.example/my-application" from
# requesting resources at "/*" when it should request them at
# "/my-application/*" instead
location = /my-application {
return 308 /my-application/;
}
location /my-application/ {
proxy_to http://htpack-addr:8080/;
proxy_intercept_errors on;
error_page 404 =200 /my-application/;
}
If you are using the handler as a library, then you may call
`handler.SetNotFound(filename)` to select a resource to return (with HTTP 200)
if a request is made for a resource that is not found. The filename must match
a packed resource, so it will be preceded with a `/` (for example it may be
`"/index.html"`).

View File

@ -6,8 +6,10 @@ require (
github.com/andybalholm/brotli v1.0.0
github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2
github.com/kr/pretty v0.1.0 // indirect
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381
github.com/spf13/cobra v0.0.5
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1
github.com/vbauerster/mpb/v4 v4.11.2
golang.org/x/sys v0.0.0-20200116001909-b77594299b42
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/yaml.v2 v2.2.2
src.lwithers.me.uk/go/htpack v1.1.5

View File

@ -1,4 +1,8 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@ -22,6 +26,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@ -38,12 +44,24 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/vbauerster/mpb/v4 v4.11.2 h1:ynkUoKzi65DZ1UsQPx7sgi/KN6G9f7br+Us2nKm35AM=
github.com/vbauerster/mpb/v4 v4.11.2/go.mod h1:jIuIRCltGJUnm6DCyPVkwjlLUk4nHTH+m4eD14CdFF0=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4=
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 h1:gZpLHxUX5BdYLA08Lj4YCJNN/jk7KtquiArPoeX0WvA=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@ -32,7 +32,6 @@ var inspectCmd = &cobra.Command{
// Inspect a packfile.
// TODO: verify etag; verify integrity of compressed data.
// TODO: skip Gzip/Brotli if not present; print ratio.
func Inspect(filename string) error {
f, err := os.Open(filename)
if err != nil {
@ -65,10 +64,46 @@ func Inspect(filename string) error {
printSize(info.Brotli.Length), info.Brotli.Offset)
}
}
inspectSummary(dir)
}
return err
}
func inspectSummary(dir *packed.Directory) {
var (
n, ngzip, nbrotli int
s, sgzip, sbrotli uint64
)
for _, f := range dir.Files {
n++
s += f.Uncompressed.Length
if f.Gzip != nil {
ngzip++
sgzip += f.Gzip.Length
}
if f.Brotli != nil {
nbrotli++
sbrotli += f.Brotli.Length
}
}
fmt.Printf("Uncompressed:\n\tFiles: %d\n\tSize: %s\n",
n, printSize(s))
if ngzip > 0 {
fmt.Printf("gzip compressed:\n\tFiles: %d (%.1f%% of total)\n"+
"\tSize: %s\n\tRatio: %.1f%%\n",
ngzip, 100*float64(ngzip)/float64(n),
printSize(sgzip), 100*float64(sgzip)/float64(s))
}
if nbrotli > 0 {
fmt.Printf("brotli compressed:\n\tFiles: %d (%.1f%% of total)\n"+
"\tSize: %s\n\tRatio: %.1f%%\n",
nbrotli, 100*float64(nbrotli)/float64(n),
printSize(sbrotli), 100*float64(sbrotli)/float64(s))
}
}
func printSize(size uint64) string {
switch {
case size < 1<<10:

View File

@ -10,6 +10,7 @@ import (
"github.com/spf13/cobra"
yaml "gopkg.in/yaml.v2"
"src.lwithers.me.uk/go/htpack/cmd/htpacker/packer"
"src.lwithers.me.uk/go/htpack/packed"
)
var packCmd = &cobra.Command{
@ -89,7 +90,8 @@ func PackFiles(c *cobra.Command, args []string, out string) error {
if err != nil {
return err
}
return packer.Pack(ftp, out)
return doPack(ftp, out)
}
func PackSpec(c *cobra.Command, spec, out string) error {
@ -103,5 +105,26 @@ func PackSpec(c *cobra.Command, spec, out string) error {
return fmt.Errorf("parsing YAML spec %s: %v", spec, err)
}
return packer.Pack(ftp, out)
return doPack(ftp, out)
}
func doPack(ftp packer.FilesToPack, out string) error {
prog := mpbProgress(ftp)
err := packer.Pack2(ftp, out, prog)
prog.Complete()
if err == nil {
fin, err := os.Open(out)
if err != nil {
return err
}
defer fin.Close()
_, dir, err := packed.Load(fin)
if err != nil {
return err
}
inspectSummary(dir)
}
return err
}

View File

@ -0,0 +1,144 @@
package main
import (
"github.com/logrusorgru/aurora"
"github.com/vbauerster/mpb/v4"
"github.com/vbauerster/mpb/v4/decor"
"src.lwithers.me.uk/go/htpack/cmd/htpacker/packer"
)
const mpbBarStyle = "[██░]"
// mpbProgress returns a new progress object that keeps the user informed via
// the visual mpb library.
func mpbProgress(ftp packer.FilesToPack) *mpbProg {
mp := new(mpbProg)
mp.un.max = len(ftp)
for _, f := range ftp {
if !f.DisableCompression && !f.DisableGzip {
mp.gzip.max++
}
if !f.DisableCompression && !f.DisableBrotli {
mp.brotli.max++
}
}
mp.p = mpb.New()
mp.un.bar = mp.p.AddBar(int64(mp.un.max),
mpb.PrependDecorators(barName("uncompressed")),
mpb.BarStyle(mpbBarStyle),
mpb.AppendDecorators(&mp.un))
if mp.gzip.max > 0 {
mp.gzip.bar = mp.p.AddBar(int64(mp.gzip.max),
mpb.PrependDecorators(barName("gzip")),
mpb.BarStyle(mpbBarStyle),
mpb.AppendDecorators(&mp.gzip))
}
if mp.brotli.max > 0 {
mp.brotli.bar = mp.p.AddBar(int64(mp.brotli.max),
mpb.PrependDecorators(barName("brotli")),
mpb.BarStyle(mpbBarStyle),
mpb.AppendDecorators(&mp.brotli))
}
return mp
}
func barName(n string) decor.Decorator {
return decor.Name(aurora.Magenta(n).String(), decor.WCSyncWidth)
}
// mpbProg is the mpb progress tracker. It has one bar for each type of
// compression, and its methods simply dispatch onto the type-specific
// bars.
type mpbProg struct {
un, gzip, brotli mpbProg1
p *mpb.Progress
}
func (mp *mpbProg) Count(_ int) {
}
func (mp *mpbProg) Begin(filename, compression string) {
switch compression {
case "uncompressed":
mp.un.Begin(filename)
case "gzip":
mp.gzip.Begin(filename)
case "brotli":
mp.brotli.Begin(filename)
default:
return
}
}
func (mp *mpbProg) End(filename, compression string) {
switch compression {
case "uncompressed":
mp.un.End(filename)
case "gzip":
mp.gzip.End(filename)
case "brotli":
mp.brotli.End(filename)
default:
return
}
}
func (mp *mpbProg) Complete() {
mp.un.Complete()
mp.gzip.Complete()
mp.brotli.Complete()
mp.p.Wait()
}
// mpbProg1 is a type-specific progress bar. In addition to holding state and
// methods for updating the bar, it also implements decor.Decor.
type mpbProg1 struct {
max int // number of items we expect
done int // number of items completed
cur []string // list of currently-packing filenames
bar *mpb.Bar
// embedding this type lets us implement decor.Decor
decor.WC
}
func (mp1 *mpbProg1) Decor(stat *decor.Statistics) string {
if stat.Completed {
return ""
}
switch len(mp1.cur) {
case 0:
return aurora.Gray(8, "(idle)").String()
case 1:
return aurora.Blue(mp1.cur[0]).String()
default:
return aurora.Sprintf(aurora.Green("%s + %d more"), aurora.Blue(mp1.cur[0]), len(mp1.cur)-1)
}
}
func (mp1 *mpbProg1) Begin(filename string) {
mp1.cur = append(mp1.cur, filename)
}
func (mp1 *mpbProg1) End(filename string) {
for i, v := range mp1.cur {
if v == filename {
mp1.cur[i] = mp1.cur[len(mp1.cur)-1]
mp1.cur = mp1.cur[:len(mp1.cur)-1]
break
}
}
mp1.done++
if mp1.bar != nil {
mp1.bar.SetCurrent(int64(mp1.done))
}
}
func (mp1 *mpbProg1) Complete() {
if mp1.bar != nil {
mp1.bar.SetTotal(int64(mp1.max), true)
}
}

View File

@ -45,6 +45,24 @@ type FileToPack struct {
DisableBrotli bool `yaml:"disable_brotli"`
}
// Progress is a callback object which reports packing progress.
type Progress interface {
// Count reports the number of items that have begun processing.
Count(n int)
// Begin denotes the processing of an input file.
Begin(filename, compression string)
// End denotes the completion of input file processing.
End(filename, compression string)
}
type ignoreProgress int
func (ignoreProgress) Count(_ int) {}
func (ignoreProgress) Begin(_, _ string) {}
func (ignoreProgress) End(_, _ string) {}
const (
// minCompressionSaving means we'll only use the compressed version of
// the file if it's at least this many bytes smaller than the original.
@ -68,8 +86,18 @@ const (
sendfileLimit = 0x7FFFF000
)
// Pack a file.
// Pack a file. Use Pack2 for progress reporting.
func Pack(filesToPack FilesToPack, outputFilename string) error {
return Pack2(filesToPack, outputFilename, nil)
}
// Pack2 will pack a file, with progress reporting. The progress interface may
// be nil.
func Pack2(filesToPack FilesToPack, outputFilename string, progress Progress) error {
if progress == nil {
progress = ignoreProgress(0)
}
finalFname, w, err := writefile.New(outputFilename)
if err != nil {
return err
@ -79,7 +107,8 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
// we use this little structure to serialise file writes below, and
// it has a couple of convenience methods for common operations
packer := packer{
w: w,
w: w,
progress: progress,
}
// write initial header (will rewrite offset/length when known)
@ -124,6 +153,7 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
Files: make(map[string]*packed.File),
}
var count int
PackingLoop:
for path, fileToPack := range filesToPack {
select {
@ -132,6 +162,8 @@ PackingLoop:
break PackingLoop
default:
packer.packFile(path, fileToPack)
count++
progress.Count(count)
}
}
@ -199,12 +231,13 @@ func compressionWorthwhile(data []byte, compressed os.FileInfo) bool {
// of compression. Unexported methods assume they are called in a context where
// the lock is not needed or already taken; exported methods take the lock.
type packer struct {
w *os.File
lock sync.Mutex
cpus chan struct{}
errors chan error
aborted chan struct{}
dir *packed.Directory
w *os.File
lock sync.Mutex
cpus chan struct{}
errors chan error
aborted chan struct{}
dir *packed.Directory
progress Progress
}
// pad will move the file write pointer to the next padding boundary. It is not
@ -312,11 +345,15 @@ func (p *packer) packFile(path string, fileToPack FileToPack) {
// list of operations on this input file that we'll carry out asynchronously
ops := []func() error{
func() error {
p.progress.Begin(fileToPack.Filename, "uncompressed")
defer p.progress.End(fileToPack.Filename, "uncompressed")
return p.Uncompressed(fileToPack.Filename, info)
},
}
if !fileToPack.DisableCompression && !fileToPack.DisableGzip {
ops = append(ops, func() error {
p.progress.Begin(fileToPack.Filename, "gzip")
defer p.progress.End(fileToPack.Filename, "gzip")
if err := p.Gzip(data, info); err != nil {
return fmt.Errorf("gzip compression of %s "+
"failed: %v", fileToPack.Filename, err)
@ -326,6 +363,8 @@ func (p *packer) packFile(path string, fileToPack FileToPack) {
}
if !fileToPack.DisableCompression && !fileToPack.DisableBrotli {
ops = append(ops, func() error {
p.progress.Begin(fileToPack.Filename, "brotli")
defer p.progress.End(fileToPack.Filename, "brotli")
if err := p.Brotli(data, info); err != nil {
return fmt.Errorf("brotli compression of %s "+
"failed: %v", fileToPack.Filename, err)

View File

@ -72,6 +72,7 @@ type Handler struct {
dir map[string]*packed.File
headers map[string]string
startTime time.Time
notFound *packed.File
}
// SetHeader allows a custom header to be set on HTTP responses. These are
@ -110,6 +111,28 @@ func (h *Handler) SetIndex(filename string) {
}
}
// SetNotFound allows overriding the returned resource when a request is made
// for a resource that does not exist. The default behaviour would be to return
// a standard HTTP 404 Not Found response; calling this function with an empty
// string will restore that behaviour.
//
// This function will return an error if the named resource is not present in
// the packfile.
func (h *Handler) SetNotFound(notFound string) error {
if notFound == "" {
h.notFound = nil
return nil
}
notFound = path.Clean(notFound)
dir := h.dir[path.Clean(notFound)]
if dir == nil {
return fmt.Errorf("no such resource %q", notFound)
}
h.notFound = dir
return nil
}
// ServeHTTP handles requests for files. It supports GET and HEAD methods, with
// anything else returning a 405. Exact path matches are required, else a 404 is
// returned.
@ -130,8 +153,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
info := h.dir[path.Clean(req.URL.Path)]
if info == nil {
http.NotFound(w, req)
return
if h.notFound == nil {
http.NotFound(w, req)
return
}
info = h.notFound
}
// set standard headers