go
/
htpack
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Further work; tidying up, begin cmdline tool

This commit is contained in:
Laurence Withers 2019-03-13 21:55:31 +00:00
parent 2a25d80249
commit 5c05a75fba
5 changed files with 180 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/htpacker/inspector.go
View File

@ -1,12 +1,35 @@
package main
import (
"errors"
"fmt"
"os"
"github.com/lwithers/htpack/internal/packed"
"github.com/spf13/cobra"
)
var inspectCmd = &cobra.Command{
Use: "inspect",
Short: "View contents of an htpack file",
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return errors.New("must specify one or more files")
}
var exitCode int
for _, filename := range args {
if err := Inspect(filename); err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n",
filename, err)
exitCode = 1
}
}
os.Exit(exitCode)
return nil
},
}
// Inspect a packfile.
// TODO: verify etag; verify integrity of compressed data.
// TODO: skip Gzip/Brotli if not present; print ratio.

28
cmd/htpacker/main.go Normal file
View File

@ -0,0 +1,28 @@
package main
import (
"fmt"
"os"
"github.com/spf13/cobra"
)
var rootCmd = &cobra.Command{
Use: "htpacker",
Short: "htpacker packs static files into a blob that can be served efficiently over HTTP",
Long: `Creates .htpack files comprising one or more static assets, and
compressed versions thereof. A YAML specification of files to pack may be
provided or generated on demand; or files and directories can be listed as
arguments.`,
}
func main() {
rootCmd.AddCommand(packCmd)
//rootCmd.AddCommand(yamlCmd)
rootCmd.AddCommand(inspectCmd)
if err := rootCmd.Execute(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}

75
cmd/htpacker/pack.go Normal file
View File

@ -0,0 +1,75 @@
package main
import (
"errors"
"fmt"
"io/ioutil"
"os"
"github.com/lwithers/htpack/packer"
"github.com/spf13/cobra"
yaml "gopkg.in/yaml.v2"
)
var packCmd = &cobra.Command{
Use: "pack",
Short: "creates a packfile from a YAML spec or set of files/dirs",
RunE: func(c *cobra.Command, args []string) error {
spec, err := c.Flags().GetString("spec")
if err != nil {
return err
}
if spec == "" {
if len(args) == 0 {
return errors.New("need --yaml, " +
"or one or more filenames")
}
err = PackFiles(c, args)
} else {
if len(args) != 0 {
return errors.New("cannot specify files " +
"when using --yaml")
}
err = PackSpec(c, spec)
}
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
return nil
},
}
func init() {
packCmd.Flags().StringP("out", "O", "",
"Output filename")
packCmd.MarkFlagRequired("out")
packCmd.Flags().StringP("spec", "y", "",
"YAML specification file (if not present, just pack files)")
packCmd.Flags().StringP("chdir", "C", "",
"Change to directory before searching for input files")
}
func PackFiles(c *cobra.Command, args []string) error {
// TODO
return errors.New("not implemented yet")
}
func PackSpec(c *cobra.Command, spec string) error {
raw, err := ioutil.ReadFile(spec)
if err != nil {
return err
}
var ftp packer.FilesToPack
if err := yaml.UnmarshalStrict(raw, &ftp); err != nil {
return fmt.Errorf("parsing YAML spec %s: %v", spec, err)
}
// TODO: chdir
out, _ := c.Flags().GetString("out")
return packer.Pack(ftp, out)
}

31
cmd/htpacker/quick_pack.go
View File

@ -1,31 +0,0 @@
package main
import (
"fmt"
"io/ioutil"
"os"
yaml "gopkg.in/yaml.v2"
)
func main() {
//if err := dopack(); err != nil {
if err := Inspect("out.htpack"); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func dopack() error {
raw, err := ioutil.ReadFile("in.yaml")
if err != nil {
return err
}
var ftp FilesToPack
if err := yaml.UnmarshalStrict(raw, &ftp); err != nil {
return err
}
return Pack(ftp, "out.htpack")
}

78
cmd/htpacker/packer.go → packer/packer.go
View File

@ -1,4 +1,4 @@
package main
package packer
import (
"bufio"
@ -16,8 +16,6 @@ import (
"github.com/lwithers/pkg/writefile"
)
// TODO: abandon packed version if no size saving
var BrotliPath string = "brotli"
type FilesToPack map[string]FileToPack
@ -37,6 +35,20 @@ type packInfo struct {
offset, len uint64
}
const (
// minCompressionSaving means we'll only use the compressed version of
// the file if it's at least this many bytes smaller than the original.
// Chosen somewhat arbitrarily; we have to add an HTTP header, and the
// decompression overhead is not zero.
minCompressionSaving = 128
// minCompressionFraction means we'll only use the compressed version of
// the file if it's at least (origSize>>minCompressionFraction) bytes
// smaller than the original. This is a guess at when the decompression
// overhead outweighs the time saved in transmission.
minCompressionFraction = 7 // i.e. files must be at least 1/128 smaller
)
// Pack a file.
func Pack(filesToPack FilesToPack, outputFilename string) error {
finalFname, outputFile, err := writefile.New(outputFilename)
@ -70,7 +82,7 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
// write the directory
if m, err = dir.Marshal(); err != nil {
// TODO: decorate
err = fmt.Errorf("marshaling directory object: %v", err)
return err
}
@ -78,7 +90,6 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
hdr.DirectoryOffset = packer.Pos()
hdr.DirectoryLength = uint64(len(m))
if _, err := packer.Write(m); err != nil {
// TODO: decorate
return err
}
@ -113,7 +124,7 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e
data, err := unix.Mmap(int(f.Fd()), 0, int(fi.Size()),
unix.PROT_READ, unix.MAP_SHARED)
if err != nil {
// TODO: decorate
err = fmt.Errorf("mmap %s: %v", fileToPack.Filename, err)
return
}
defer unix.Munmap(data)
@ -129,8 +140,7 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e
Offset: packer.Pos(),
Length: uint64(len(data)),
}
if _, err = packer.CopyFrom(f); err != nil {
// TODO: decorate
if _, err = packer.CopyFrom(f, fi); err != nil {
return
}
info.Uncompressed = fileData
@ -147,11 +157,14 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e
fileData = &packed.FileData{
Offset: packer.Pos(),
}
fileData.Length, err = packOneGzip(packer, data)
fileData.Length, err = packOneGzip(packer, data,
info.Uncompressed.Length)
if err != nil {
return
}
info.Gzip = fileData
if fileData.Length > 0 {
info.Gzip = fileData
}
}
// brotli compression
@ -162,11 +175,14 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e
fileData = &packed.FileData{
Offset: packer.Pos(),
}
fileData.Length, err = packOneBrotli(packer, fileToPack.Filename)
fileData.Length, err = packOneBrotli(packer,
fileToPack.Filename, info.Uncompressed.Length)
if err != nil {
return
}
info.Brotli = fileData
if fileData.Length > 0 {
info.Brotli = fileData
}
}
return
@ -178,7 +194,8 @@ func etag(in []byte) string {
return fmt.Sprintf(`"1--%x"`, h.Sum(nil))
}
func packOneGzip(packer *packWriter, data []byte) (uint64, error) {
func packOneGzip(packer *packWriter, data []byte, uncompressedSize uint64,
) (uint64, error) {
// write via temporary file
tmpfile, err := ioutil.TempFile("", "")
if err != nil {
@ -202,10 +219,11 @@ func packOneGzip(packer *packWriter, data []byte) (uint64, error) {
}
// copy into packfile
return packer.CopyFrom(tmpfile)
return packer.CopyIfSaving(tmpfile, uncompressedSize)
}
func packOneBrotli(packer *packWriter, filename string) (uint64, error) {
func packOneBrotli(packer *packWriter, filename string, uncompressedSize uint64,
) (uint64, error) {
// write via temporary file
tmpfile, err := ioutil.TempFile("", "")
if err != nil {
@ -219,13 +237,12 @@ func packOneBrotli(packer *packWriter, filename string) (uint64, error) {
"--output", tmpfile.Name())
out, err := cmd.CombinedOutput()
if err != nil {
// TODO: decorate
_ = out
err = fmt.Errorf("brotli: %v (process reported: %s)", err, out)
return 0, err
}
// copy into packfile
return packer.CopyFrom(tmpfile)
return packer.CopyIfSaving(tmpfile, uncompressedSize)
}
type packWriter struct {
@ -272,7 +289,7 @@ func (pw *packWriter) Pad() error {
return pw.err
}
func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
func (pw *packWriter) CopyIfSaving(in *os.File, uncompressedSize uint64) (uint64, error) {
if pw.err != nil {
return 0, pw.err
}
@ -282,8 +299,22 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
pw.err = err
return 0, pw.err
}
sz := uint64(fi.Size())
fmt.Fprintf(os.Stderr, "[DEBUG] in size=%d\n", fi.Size())
if sz+minCompressionSaving > uncompressedSize {
return 0, nil
}
if sz+(uncompressedSize>>minCompressionFraction) > uncompressedSize {
return 0, nil
}
return pw.CopyFrom(in, fi)
}
func (pw *packWriter) CopyFrom(in *os.File, fi os.FileInfo) (uint64, error) {
if pw.err != nil {
return 0, pw.err
}
var off int64
remain := fi.Size()
@ -295,12 +326,11 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
amt = int(remain)
}
amt, err = unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt)
fmt.Fprintf(os.Stderr, "[DEBUG] sendfile=%d [off now %d]\n", amt, off)
amt, err := unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt)
remain -= int64(amt)
//off += int64(amt)
if err != nil {
pw.err = err
pw.err = fmt.Errorf("sendfile (copying data to "+
"htpack): %v", err)
return uint64(off), pw.err
}
}