Finish porting child modules to src.lwithers.me.uk
This commit is contained in:
parent
1a37903d98
commit
984639d475
|
|
@ -1,14 +1,15 @@
|
|||
module github.com/lwithers/htpack/cmd/htpacker
|
||||
module src.lwithers.me.uk/go/htpack/cmd/htpacker
|
||||
|
||||
go 1.12
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.0.0
|
||||
github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/lwithers/htpack v1.1.2
|
||||
github.com/lwithers/pkg v1.2.1
|
||||
github.com/spf13/cobra v0.0.5
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
gopkg.in/yaml.v2 v2.2.2
|
||||
src.lwithers.me.uk/go/htpack v1.1.5
|
||||
src.lwithers.me.uk/go/writefile v1.0.1
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
|
||||
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
|
|
@ -20,10 +22,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
|
|||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lwithers/htpack v1.1.2 h1:fWHNYBQQT51jSTvoOj1xJvD68NSbZq23v9ygi1sU2yg=
|
||||
github.com/lwithers/htpack v1.1.2/go.mod h1:4dNHChTcK0SzOTVnFt4b0SuK7OMSo8Ge7o1XXYV4xUk=
|
||||
github.com/lwithers/pkg v1.2.1 h1:KNnZFGv0iyduc+uUF5UB8vDyr2ofRq930cVKqrpQulY=
|
||||
github.com/lwithers/pkg v1.2.1/go.mod h1:0CRdDnVCqIa5uaIs1u8Gmwl3M7sm181QmSmVVaPTZUo=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
|
|
@ -42,12 +40,10 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
|
|||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/sys v0.0.0-20180924175946-90868a75fefd/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190415081028-16da32be82c5 h1:UMbOtg4ZL2GyTAolLE9QfNvzskWvFkI935Z98i9moXA=
|
||||
golang.org/x/sys v0.0.0-20190415081028-16da32be82c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 h1:gZpLHxUX5BdYLA08Lj4YCJNN/jk7KtquiArPoeX0WvA=
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
|
@ -55,3 +51,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
|
|||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
src.lwithers.me.uk/go/htpack v1.1.5 h1:2JzgqLZ1ROYc53+96NezfJ3S9fkwHZNd6QgJhMXnlSE=
|
||||
src.lwithers.me.uk/go/htpack v1.1.5/go.mod h1:JWofpm01RJbCTIyKfIPftUsxk6KlFkrYwyHgCVdKY+s=
|
||||
src.lwithers.me.uk/go/writefile v1.0.1 h1:bwBGtvyZfCxFIM14e1aYgJWlZuowKkwJx53OJlUPd0s=
|
||||
src.lwithers.me.uk/go/writefile v1.0.1/go.mod h1:NahlmRCtB7kg4ai+zHZgxXdUs+MR8VqWG8mql35TsxA=
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/lwithers/htpack/packed"
|
||||
"github.com/spf13/cobra"
|
||||
"src.lwithers.me.uk/go/htpack/packed"
|
||||
)
|
||||
|
||||
var inspectCmd = &cobra.Command{
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/lwithers/htpack/cmd/htpacker/packer"
|
||||
"github.com/spf13/cobra"
|
||||
yaml "gopkg.in/yaml.v2"
|
||||
"src.lwithers.me.uk/go/htpack/cmd/htpacker/packer"
|
||||
)
|
||||
|
||||
var packCmd = &cobra.Command{
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
/*
|
||||
Package packer implements the core packing functionality. It is designed to be
|
||||
used by a wrapper program (CLI etc.).
|
||||
*/
|
||||
package packer
|
||||
|
||||
import (
|
||||
|
|
@ -7,32 +11,38 @@ import (
|
|||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"github.com/andybalholm/brotli"
|
||||
"github.com/foobaz/go-zopfli/zopfli"
|
||||
"github.com/lwithers/htpack/packed"
|
||||
"github.com/lwithers/pkg/writefile"
|
||||
"golang.org/x/sys/unix"
|
||||
"src.lwithers.me.uk/go/htpack/packed"
|
||||
"src.lwithers.me.uk/go/writefile"
|
||||
)
|
||||
|
||||
var BrotliPath string = "brotli"
|
||||
|
||||
// FilesToPack is the set of files which will be incorporated into the packfile.
|
||||
// The key is the path at which the file will be served, and the value gives the
|
||||
// disk filename as well as headers / options.
|
||||
type FilesToPack map[string]FileToPack
|
||||
|
||||
// FileToPack contains the headers / options for a file which is about to be
|
||||
// packed.
|
||||
type FileToPack struct {
|
||||
Filename string `yaml:"filename"`
|
||||
ContentType string `yaml:"content_type"`
|
||||
DisableCompression bool `yaml:"disable_compression"`
|
||||
DisableGzip bool `yaml:"disable_gzip"`
|
||||
DisableBrotli bool `yaml:"disable_brotli"`
|
||||
// Filename is the path to the file on disk (relative or absolute).
|
||||
Filename string `yaml:"filename"`
|
||||
|
||||
uncompressed, gzip, brotli packInfo
|
||||
}
|
||||
// ContentType is used as the Content-Type header for the source data.
|
||||
ContentType string `yaml:"content_type"`
|
||||
|
||||
type packInfo struct {
|
||||
present bool
|
||||
offset, len uint64
|
||||
// DisableCompression can be set to skip any compression for this file.
|
||||
DisableCompression bool `yaml:"disable_compression"`
|
||||
|
||||
// DisableGzip can be set to skip gzip compression for this file.
|
||||
DisableGzip bool `yaml:"disable_gzip"`
|
||||
|
||||
// DisableBrotli can be set to skip brotli compression for this file.
|
||||
DisableBrotli bool `yaml:"disable_brotli"`
|
||||
}
|
||||
|
||||
const (
|
||||
|
|
@ -47,16 +57,30 @@ const (
|
|||
// smaller than the original. This is a guess at when the decompression
|
||||
// overhead outweighs the time saved in transmission.
|
||||
minCompressionFraction = 7 // i.e. files must be at least 1/128 smaller
|
||||
|
||||
// padWidth is the padding alignment size expressed as a power of 2.
|
||||
// The value 12 (i.e. 4096 bytes) is chosen to align with a common
|
||||
// page size and filesystem block size.
|
||||
padWidth = 12
|
||||
|
||||
// sendfileLimit is the number of bytes we can transfer through a single
|
||||
// sendfile(2) call. This value is from the man page.
|
||||
sendfileLimit = 0x7FFFF000
|
||||
)
|
||||
|
||||
// Pack a file.
|
||||
func Pack(filesToPack FilesToPack, outputFilename string) error {
|
||||
finalFname, outputFile, err := writefile.New(outputFilename)
|
||||
finalFname, w, err := writefile.New(outputFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer writefile.Abort(outputFile)
|
||||
packer := &packWriter{f: outputFile}
|
||||
defer writefile.Abort(w)
|
||||
|
||||
// we use this little structure to serialise file writes below, and
|
||||
// it has a couple of convenience methods for common operations
|
||||
packer := packer{
|
||||
w: w,
|
||||
}
|
||||
|
||||
// write initial header (will rewrite offset/length when known)
|
||||
hdr := &packed.Header{
|
||||
|
|
@ -65,127 +89,91 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
|
|||
DirectoryOffset: 1,
|
||||
DirectoryLength: 1,
|
||||
}
|
||||
m, _ := hdr.Marshal()
|
||||
packer.Write(m)
|
||||
m, err := hdr.Marshal()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal header (%T): %v", hdr, err)
|
||||
}
|
||||
if _, err = w.Write(m); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = packer.pad(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dir := packed.Directory{
|
||||
// Channel to limit number of CPU-bound goroutines. One token is written
|
||||
// to the channel for each active worker; since the channel is bounded,
|
||||
// further writes will block at the limit. As workers complete, they
|
||||
// consume a token from the channel.
|
||||
nCPU := runtime.NumCPU() + 2 // +2 for I/O bound portions
|
||||
if nCPU < 4 {
|
||||
nCPU = 4
|
||||
}
|
||||
packer.cpus = make(chan struct{}, nCPU)
|
||||
|
||||
// Channel to report worker errors. Writes should be non-blocking. If
|
||||
// your error is dropped, don't worry, an earlier error will be
|
||||
// reported.
|
||||
packer.errors = make(chan error, 1)
|
||||
|
||||
// Channel to abort further operations. It should be closed to abort.
|
||||
// The closer should be the one who writes onto packer.errors.
|
||||
packer.aborted = make(chan struct{})
|
||||
|
||||
// write the packed files, storing info for the directory structure
|
||||
packer.dir = &packed.Directory{
|
||||
Files: make(map[string]*packed.File),
|
||||
}
|
||||
|
||||
PackingLoop:
|
||||
for path, fileToPack := range filesToPack {
|
||||
info, err := packOne(packer, fileToPack)
|
||||
if err != nil {
|
||||
return err
|
||||
select {
|
||||
case <-packer.aborted:
|
||||
// a worker reported an error; break out of loop early
|
||||
break PackingLoop
|
||||
default:
|
||||
packer.packFile(path, fileToPack)
|
||||
}
|
||||
dir.Files[path] = &info
|
||||
}
|
||||
|
||||
// wait for all goroutines to complete
|
||||
for n := 0; n < nCPU; n++ {
|
||||
packer.cpus <- struct{}{}
|
||||
}
|
||||
|
||||
// check whether any of the just-completed goroutines returned an error
|
||||
select {
|
||||
case err = <-packer.errors:
|
||||
return err
|
||||
default:
|
||||
}
|
||||
|
||||
// write the directory
|
||||
if m, err = dir.Marshal(); err != nil {
|
||||
err = fmt.Errorf("marshaling directory object: %v", err)
|
||||
if m, err = packer.dir.Marshal(); err != nil {
|
||||
err = fmt.Errorf("failed to marshal directory object (%T): %v",
|
||||
packer.dir, err)
|
||||
return err
|
||||
}
|
||||
dirOffset, err := w.Seek(0, os.SEEK_CUR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
packer.Pad()
|
||||
hdr.DirectoryOffset = packer.Pos()
|
||||
if _, err := w.Write(m); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// now modify the header at the start of the file
|
||||
hdr.DirectoryOffset = uint64(dirOffset)
|
||||
hdr.DirectoryLength = uint64(len(m))
|
||||
if _, err := packer.Write(m); err != nil {
|
||||
return err
|
||||
if m, err = hdr.Marshal(); err != nil {
|
||||
return fmt.Errorf("failed to marshal header (%T): %v", hdr, err)
|
||||
}
|
||||
|
||||
// write header at start of file
|
||||
m, _ = hdr.Marshal()
|
||||
if _, err = outputFile.WriteAt(m, 0); err != nil {
|
||||
if _, err = w.WriteAt(m, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// all done!
|
||||
return writefile.Commit(finalFname, outputFile)
|
||||
}
|
||||
|
||||
func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err error) {
|
||||
// implementation detail: write files at a page boundary
|
||||
if err = packer.Pad(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// open and mmap input file
|
||||
f, err := os.Open(fileToPack.Filename)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
data, err := unix.Mmap(int(f.Fd()), 0, int(fi.Size()),
|
||||
unix.PROT_READ, unix.MAP_SHARED)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("mmap %s: %v", fileToPack.Filename, err)
|
||||
return
|
||||
}
|
||||
defer unix.Munmap(data)
|
||||
|
||||
info.Etag = etag(data)
|
||||
info.ContentType = fileToPack.ContentType
|
||||
if info.ContentType == "" {
|
||||
info.ContentType = http.DetectContentType(data)
|
||||
}
|
||||
|
||||
// copy the uncompressed version
|
||||
fileData := &packed.FileData{
|
||||
Offset: packer.Pos(),
|
||||
Length: uint64(len(data)),
|
||||
}
|
||||
if _, err = packer.CopyFrom(f, fi); err != nil {
|
||||
return
|
||||
}
|
||||
info.Uncompressed = fileData
|
||||
|
||||
if fileToPack.DisableCompression {
|
||||
return
|
||||
}
|
||||
|
||||
// gzip compression
|
||||
if !fileToPack.DisableGzip {
|
||||
if err = packer.Pad(); err != nil {
|
||||
return
|
||||
}
|
||||
fileData = &packed.FileData{
|
||||
Offset: packer.Pos(),
|
||||
}
|
||||
fileData.Length, err = packOneGzip(packer, data,
|
||||
info.Uncompressed.Length)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if fileData.Length > 0 {
|
||||
info.Gzip = fileData
|
||||
}
|
||||
}
|
||||
|
||||
// brotli compression
|
||||
if BrotliPath != "" && !fileToPack.DisableBrotli {
|
||||
if err = packer.Pad(); err != nil {
|
||||
return
|
||||
}
|
||||
fileData = &packed.FileData{
|
||||
Offset: packer.Pos(),
|
||||
}
|
||||
fileData.Length, err = packOneBrotli(packer,
|
||||
fileToPack.Filename, info.Uncompressed.Length)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if fileData.Length > 0 {
|
||||
info.Brotli = fileData
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
return writefile.Commit(finalFname, w)
|
||||
}
|
||||
|
||||
func etag(in []byte) string {
|
||||
|
|
@ -194,12 +182,219 @@ func etag(in []byte) string {
|
|||
return fmt.Sprintf(`"1--%x"`, h.Sum(nil))
|
||||
}
|
||||
|
||||
func packOneGzip(packer *packWriter, data []byte, uncompressedSize uint64,
|
||||
) (uint64, error) {
|
||||
func compressionWorthwhile(data []byte, compressed os.FileInfo) bool {
|
||||
uncompressedSize := uint64(len(data))
|
||||
sz := uint64(compressed.Size())
|
||||
|
||||
switch {
|
||||
case sz+minCompressionSaving > uncompressedSize,
|
||||
sz+(uncompressedSize>>minCompressionFraction) > uncompressedSize:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// packer packs input files into the output file. It has methods for each type
|
||||
// of compression. Unexported methods assume they are called in a context where
|
||||
// the lock is not needed or already taken; exported methods take the lock.
|
||||
type packer struct {
|
||||
w *os.File
|
||||
lock sync.Mutex
|
||||
cpus chan struct{}
|
||||
errors chan error
|
||||
aborted chan struct{}
|
||||
dir *packed.Directory
|
||||
}
|
||||
|
||||
// pad will move the file write pointer to the next padding boundary. It is not
|
||||
// concurrency safe.
|
||||
func (p *packer) pad() error {
|
||||
pos, err := p.w.Seek(0, os.SEEK_CUR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pos &= (1 << padWidth) - 1
|
||||
if pos == 0 { // already aligned
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err = p.w.Seek((1<<padWidth)-pos, os.SEEK_CUR)
|
||||
return err
|
||||
}
|
||||
|
||||
// appendPath will copy file data from srcPath and append it to the output file. The
|
||||
// offset and length are stored in ‘data’ on success. It is not concurrency safe.
|
||||
func (p *packer) appendPath(srcPath string, data *packed.FileData) error {
|
||||
// open the input file and grab its length
|
||||
in, err := os.Open(srcPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
fi, err := in.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// copy in the file data
|
||||
return p.appendFile(in, fi.Size(), data)
|
||||
}
|
||||
|
||||
// appendFile will copy file data from src and append it to the output file. The
|
||||
// offset and length are stored in ‘data’ on success. It is not concurrency safe.
|
||||
func (p *packer) appendFile(src *os.File, srcLen int64, data *packed.FileData) error {
|
||||
// retrieve current file position and store in data.Offset
|
||||
off, err := p.w.Seek(0, os.SEEK_CUR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data.Length = uint64(srcLen)
|
||||
data.Offset = uint64(off)
|
||||
|
||||
// copy in the file data
|
||||
remain := srcLen
|
||||
off = 0
|
||||
for remain > 0 {
|
||||
var amt int
|
||||
if remain > sendfileLimit {
|
||||
amt = sendfileLimit
|
||||
} else {
|
||||
amt = int(remain)
|
||||
}
|
||||
|
||||
amt, err := unix.Sendfile(int(p.w.Fd()), int(src.Fd()), &off, amt)
|
||||
remain -= int64(amt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sendfile (copying data to "+
|
||||
"htpack): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// leave output file padded to next boundary
|
||||
return p.pad()
|
||||
}
|
||||
|
||||
func (p *packer) packFile(path string, fileToPack FileToPack) {
|
||||
// open and mmap input file
|
||||
f, err := os.Open(fileToPack.Filename)
|
||||
if err != nil {
|
||||
p.Abort(err)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
p.Abort(err)
|
||||
return
|
||||
}
|
||||
|
||||
data, err := unix.Mmap(int(f.Fd()), 0, int(fi.Size()),
|
||||
unix.PROT_READ, unix.MAP_SHARED)
|
||||
if err != nil {
|
||||
p.Abort(fmt.Errorf("mmap %s: %v", fileToPack.Filename, err))
|
||||
return
|
||||
}
|
||||
|
||||
// prepare initial directory entry
|
||||
info := &packed.File{
|
||||
Etag: etag(data),
|
||||
ContentType: fileToPack.ContentType,
|
||||
}
|
||||
if info.ContentType == "" {
|
||||
info.ContentType = http.DetectContentType(data)
|
||||
}
|
||||
p.dir.Files[path] = info // NB: this part is not concurrent, so no mutex
|
||||
|
||||
// list of operations on this input file that we'll carry out asynchronously
|
||||
ops := []func() error{
|
||||
func() error {
|
||||
return p.Uncompressed(fileToPack.Filename, info)
|
||||
},
|
||||
}
|
||||
if !fileToPack.DisableCompression && !fileToPack.DisableGzip {
|
||||
ops = append(ops, func() error {
|
||||
if err := p.Gzip(data, info); err != nil {
|
||||
return fmt.Errorf("gzip compression of %s "+
|
||||
"failed: %v", fileToPack.Filename, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if !fileToPack.DisableCompression && !fileToPack.DisableBrotli {
|
||||
ops = append(ops, func() error {
|
||||
if err := p.Brotli(data, info); err != nil {
|
||||
return fmt.Errorf("brotli compression of %s "+
|
||||
"failed: %v", fileToPack.Filename, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// we have multiple operations on the file, and we need to wait for
|
||||
// them all to complete before munmap()
|
||||
wg := new(sync.WaitGroup)
|
||||
wg.Add(len(ops))
|
||||
go func() {
|
||||
wg.Wait()
|
||||
unix.Munmap(data)
|
||||
}()
|
||||
|
||||
for _, op := range ops {
|
||||
select {
|
||||
case <-p.aborted:
|
||||
// skip the operation
|
||||
wg.Done()
|
||||
|
||||
case p.cpus <- struct{}{}:
|
||||
go func(op func() error) {
|
||||
if err := op(); err != nil {
|
||||
p.Abort(err)
|
||||
}
|
||||
// release CPU token
|
||||
<-p.cpus
|
||||
wg.Done()
|
||||
}(op)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Abort records that an error occurred and records it onto the errors channel.
|
||||
// It signals workers to abort by closed the aborted channel. If called
|
||||
// multiple times, only one error will be recorded, and the aborted channel will
|
||||
// only be closed once.
|
||||
func (p *packer) Abort(err error) {
|
||||
select {
|
||||
case p.errors <- err:
|
||||
// only one error can be written to this channel, so the write
|
||||
// acts as a lock to ensure only a single close operation takes
|
||||
// place
|
||||
close(p.aborted)
|
||||
default:
|
||||
// errors channel was already written, so we're already aborted
|
||||
}
|
||||
}
|
||||
|
||||
// Uncompressed copies in an uncompressed file.
|
||||
func (p *packer) Uncompressed(srcPath string, dir *packed.File) error {
|
||||
dir.Uncompressed = new(packed.FileData)
|
||||
p.lock.Lock()
|
||||
defer p.lock.Unlock()
|
||||
return p.appendPath(srcPath, dir.Uncompressed)
|
||||
}
|
||||
|
||||
// Gzip will gzip input data to a temporary file, and then append that to the
|
||||
// output file.
|
||||
func (p *packer) Gzip(data []byte, dir *packed.File) error {
|
||||
// write via temporary file
|
||||
tmpfile, err := ioutil.TempFile("", "")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmpfile.Name())
|
||||
defer tmpfile.Close()
|
||||
|
|
@ -212,127 +407,66 @@ func packOneGzip(packer *packWriter, data []byte, uncompressedSize uint64,
|
|||
|
||||
buf := bufio.NewWriter(tmpfile)
|
||||
if err = zopfli.GzipCompress(&opts, data, buf); err != nil {
|
||||
return 0, err
|
||||
return err
|
||||
}
|
||||
if err = buf.Flush(); err != nil {
|
||||
return 0, err
|
||||
return err
|
||||
}
|
||||
|
||||
// copy into packfile
|
||||
return packer.CopyIfSaving(tmpfile, uncompressedSize)
|
||||
// grab file length, evaluate whether compression is worth it
|
||||
fi, err := tmpfile.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !compressionWorthwhile(data, fi) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// save the compressed data
|
||||
dir.Gzip = new(packed.FileData)
|
||||
p.lock.Lock()
|
||||
defer p.lock.Unlock()
|
||||
return p.appendFile(tmpfile, fi.Size(), dir.Gzip)
|
||||
}
|
||||
|
||||
func packOneBrotli(packer *packWriter, filename string, uncompressedSize uint64,
|
||||
) (uint64, error) {
|
||||
// Brotli will compress input data to a temporary file, and then append that to
|
||||
// the output file.
|
||||
func (p *packer) Brotli(data []byte, dir *packed.File) error {
|
||||
// write via temporary file
|
||||
tmpfile, err := ioutil.TempFile("", "")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmpfile.Name())
|
||||
defer tmpfile.Close()
|
||||
|
||||
// compress via commandline
|
||||
cmd := exec.Command(BrotliPath, filename, "--output", tmpfile.Name())
|
||||
out, err := cmd.CombinedOutput()
|
||||
// compress
|
||||
buf := bufio.NewWriter(tmpfile)
|
||||
comp := brotli.NewWriterOptions(buf, brotli.WriterOptions{
|
||||
Quality: 11,
|
||||
})
|
||||
if _, err = comp.Write(data); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = comp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = buf.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// grab file length, evaluate whether compression is worth it
|
||||
fi, err := tmpfile.Stat()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("brotli: %v (process reported: %s)", err, out)
|
||||
return 0, err
|
||||
return err
|
||||
}
|
||||
if !compressionWorthwhile(data, fi) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// copy into packfile
|
||||
return packer.CopyIfSaving(tmpfile, uncompressedSize)
|
||||
}
|
||||
|
||||
type packWriter struct {
|
||||
f *os.File
|
||||
err error
|
||||
}
|
||||
|
||||
func (pw *packWriter) Write(buf []byte) (int, error) {
|
||||
if pw.err != nil {
|
||||
return 0, pw.err
|
||||
}
|
||||
n, err := pw.f.Write(buf)
|
||||
pw.err = err
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (pw *packWriter) Pos() uint64 {
|
||||
pos, err := pw.f.Seek(0, os.SEEK_CUR)
|
||||
if err != nil {
|
||||
pw.err = err
|
||||
}
|
||||
return uint64(pos)
|
||||
}
|
||||
|
||||
func (pw *packWriter) Pad() error {
|
||||
if pw.err != nil {
|
||||
return pw.err
|
||||
}
|
||||
|
||||
pos, err := pw.f.Seek(0, os.SEEK_CUR)
|
||||
if err != nil {
|
||||
pw.err = err
|
||||
return pw.err
|
||||
}
|
||||
|
||||
pos &= 0xFFF
|
||||
if pos == 0 {
|
||||
return pw.err
|
||||
}
|
||||
|
||||
if _, err = pw.f.Seek(4096-pos, os.SEEK_CUR); err != nil {
|
||||
pw.err = err
|
||||
}
|
||||
return pw.err
|
||||
}
|
||||
|
||||
func (pw *packWriter) CopyIfSaving(in *os.File, uncompressedSize uint64) (uint64, error) {
|
||||
if pw.err != nil {
|
||||
return 0, pw.err
|
||||
}
|
||||
|
||||
fi, err := in.Stat()
|
||||
if err != nil {
|
||||
pw.err = err
|
||||
return 0, pw.err
|
||||
}
|
||||
sz := uint64(fi.Size())
|
||||
|
||||
if sz+minCompressionSaving > uncompressedSize {
|
||||
return 0, nil
|
||||
}
|
||||
if sz+(uncompressedSize>>minCompressionFraction) > uncompressedSize {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
return pw.CopyFrom(in, fi)
|
||||
}
|
||||
|
||||
func (pw *packWriter) CopyFrom(in *os.File, fi os.FileInfo) (uint64, error) {
|
||||
if pw.err != nil {
|
||||
return 0, pw.err
|
||||
}
|
||||
|
||||
var off int64
|
||||
remain := fi.Size()
|
||||
for remain > 0 {
|
||||
var amt int
|
||||
if remain > (1 << 30) {
|
||||
amt = (1 << 30)
|
||||
} else {
|
||||
amt = int(remain)
|
||||
}
|
||||
|
||||
amt, err := unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt)
|
||||
remain -= int64(amt)
|
||||
if err != nil {
|
||||
pw.err = fmt.Errorf("sendfile (copying data to "+
|
||||
"htpack): %v", err)
|
||||
return uint64(off), pw.err
|
||||
}
|
||||
}
|
||||
|
||||
return uint64(off), nil
|
||||
// save the compressed data
|
||||
dir.Brotli = new(packed.FileData)
|
||||
p.lock.Lock()
|
||||
defer p.lock.Unlock()
|
||||
return p.appendFile(tmpfile, fi.Size(), dir.Brotli)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,9 +9,9 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/lwithers/htpack/cmd/htpacker/packer"
|
||||
"github.com/spf13/cobra"
|
||||
yaml "gopkg.in/yaml.v2"
|
||||
"src.lwithers.me.uk/go/htpack/cmd/htpacker/packer"
|
||||
)
|
||||
|
||||
var yamlCmd = &cobra.Command{
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
module github.com/lwithers/htpack/cmd/packserver
|
||||
module src.lwithers.me.uk/go/htpack/cmd/packserver
|
||||
|
||||
go 1.12
|
||||
go 1.13
|
||||
|
||||
require (
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/lwithers/htpack v1.1.4
|
||||
github.com/spf13/cobra v0.0.3
|
||||
github.com/spf13/pflag v1.0.3 // indirect
|
||||
github.com/spf13/cobra v0.0.5
|
||||
src.lwithers.me.uk/go/htpack v1.1.5
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,17 +1,42 @@
|
|||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/lwithers/htpack v1.1.3 h1:+BOyJnQ+m0eVzxKeVoc0oG48ALktihnqwtxxzGfFzsw=
|
||||
github.com/lwithers/htpack v1.1.3/go.mod h1:4dNHChTcK0SzOTVnFt4b0SuK7OMSo8Ge7o1XXYV4xUk=
|
||||
github.com/lwithers/htpack v1.1.4 h1:aKXWg4c9WxxxFJeUyZjqSdEaDF+Y/yD85nTnHpEjWW0=
|
||||
github.com/lwithers/htpack v1.1.4/go.mod h1:4dNHChTcK0SzOTVnFt4b0SuK7OMSo8Ge7o1XXYV4xUk=
|
||||
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
|
||||
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
golang.org/x/sys v0.0.0-20190415081028-16da32be82c5 h1:UMbOtg4ZL2GyTAolLE9QfNvzskWvFkI935Z98i9moXA=
|
||||
golang.org/x/sys v0.0.0-20190415081028-16da32be82c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 h1:gZpLHxUX5BdYLA08Lj4YCJNN/jk7KtquiArPoeX0WvA=
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
src.lwithers.me.uk/go/htpack v1.1.5 h1:2JzgqLZ1ROYc53+96NezfJ3S9fkwHZNd6QgJhMXnlSE=
|
||||
src.lwithers.me.uk/go/htpack v1.1.5/go.mod h1:JWofpm01RJbCTIyKfIPftUsxk6KlFkrYwyHgCVdKY+s=
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/lwithers/htpack"
|
||||
"github.com/spf13/cobra"
|
||||
"src.lwithers.me.uk/go/htpack"
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
|
|
|
|||
Loading…
Reference in New Issue