diff --git a/cmd/htpacker/inspector.go b/cmd/htpacker/inspector.go index c89130f..81da2dd 100644 --- a/cmd/htpacker/inspector.go +++ b/cmd/htpacker/inspector.go @@ -1,12 +1,35 @@ package main import ( + "errors" "fmt" "os" "github.com/lwithers/htpack/internal/packed" + "github.com/spf13/cobra" ) +var inspectCmd = &cobra.Command{ + Use: "inspect", + Short: "View contents of an htpack file", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("must specify one or more files") + } + + var exitCode int + for _, filename := range args { + if err := Inspect(filename); err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", + filename, err) + exitCode = 1 + } + } + os.Exit(exitCode) + return nil + }, +} + // Inspect a packfile. // TODO: verify etag; verify integrity of compressed data. // TODO: skip Gzip/Brotli if not present; print ratio. diff --git a/cmd/htpacker/main.go b/cmd/htpacker/main.go new file mode 100644 index 0000000..ff8c6df --- /dev/null +++ b/cmd/htpacker/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "htpacker", + Short: "htpacker packs static files into a blob that can be served efficiently over HTTP", + Long: `Creates .htpack files comprising one or more static assets, and +compressed versions thereof. A YAML specification of files to pack may be +provided or generated on demand; or files and directories can be listed as +arguments.`, +} + +func main() { + rootCmd.AddCommand(packCmd) + //rootCmd.AddCommand(yamlCmd) + rootCmd.AddCommand(inspectCmd) + + if err := rootCmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/htpacker/pack.go b/cmd/htpacker/pack.go new file mode 100644 index 0000000..ea14d26 --- /dev/null +++ b/cmd/htpacker/pack.go @@ -0,0 +1,75 @@ +package main + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + + "github.com/lwithers/htpack/packer" + "github.com/spf13/cobra" + yaml "gopkg.in/yaml.v2" +) + +var packCmd = &cobra.Command{ + Use: "pack", + Short: "creates a packfile from a YAML spec or set of files/dirs", + RunE: func(c *cobra.Command, args []string) error { + spec, err := c.Flags().GetString("spec") + if err != nil { + return err + } + + if spec == "" { + if len(args) == 0 { + return errors.New("need --yaml, " + + "or one or more filenames") + } + err = PackFiles(c, args) + } else { + if len(args) != 0 { + return errors.New("cannot specify files " + + "when using --yaml") + } + err = PackSpec(c, spec) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + return nil + }, +} + +func init() { + packCmd.Flags().StringP("out", "O", "", + "Output filename") + packCmd.MarkFlagRequired("out") + packCmd.Flags().StringP("spec", "y", "", + "YAML specification file (if not present, just pack files)") + packCmd.Flags().StringP("chdir", "C", "", + "Change to directory before searching for input files") +} + +func PackFiles(c *cobra.Command, args []string) error { + // TODO + return errors.New("not implemented yet") +} + +func PackSpec(c *cobra.Command, spec string) error { + raw, err := ioutil.ReadFile(spec) + if err != nil { + return err + } + + var ftp packer.FilesToPack + if err := yaml.UnmarshalStrict(raw, &ftp); err != nil { + return fmt.Errorf("parsing YAML spec %s: %v", spec, err) + } + + // TODO: chdir + + out, _ := c.Flags().GetString("out") + return packer.Pack(ftp, out) +} diff --git a/cmd/htpacker/quick_pack.go b/cmd/htpacker/quick_pack.go deleted file mode 100644 index f851e15..0000000 --- a/cmd/htpacker/quick_pack.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "fmt" - "io/ioutil" - "os" - - yaml "gopkg.in/yaml.v2" -) - -func main() { - //if err := dopack(); err != nil { - if err := Inspect("out.htpack"); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -func dopack() error { - raw, err := ioutil.ReadFile("in.yaml") - if err != nil { - return err - } - - var ftp FilesToPack - if err := yaml.UnmarshalStrict(raw, &ftp); err != nil { - return err - } - - return Pack(ftp, "out.htpack") -} diff --git a/cmd/htpacker/packer.go b/packer/packer.go similarity index 71% rename from cmd/htpacker/packer.go rename to packer/packer.go index 213f938..aefdf60 100644 --- a/cmd/htpacker/packer.go +++ b/packer/packer.go @@ -1,4 +1,4 @@ -package main +package packer import ( "bufio" @@ -16,8 +16,6 @@ import ( "github.com/lwithers/pkg/writefile" ) -// TODO: abandon packed version if no size saving - var BrotliPath string = "brotli" type FilesToPack map[string]FileToPack @@ -37,6 +35,20 @@ type packInfo struct { offset, len uint64 } +const ( + // minCompressionSaving means we'll only use the compressed version of + // the file if it's at least this many bytes smaller than the original. + // Chosen somewhat arbitrarily; we have to add an HTTP header, and the + // decompression overhead is not zero. + minCompressionSaving = 128 + + // minCompressionFraction means we'll only use the compressed version of + // the file if it's at least (origSize>>minCompressionFraction) bytes + // smaller than the original. This is a guess at when the decompression + // overhead outweighs the time saved in transmission. + minCompressionFraction = 7 // i.e. files must be at least 1/128 smaller +) + // Pack a file. func Pack(filesToPack FilesToPack, outputFilename string) error { finalFname, outputFile, err := writefile.New(outputFilename) @@ -70,7 +82,7 @@ func Pack(filesToPack FilesToPack, outputFilename string) error { // write the directory if m, err = dir.Marshal(); err != nil { - // TODO: decorate + err = fmt.Errorf("marshaling directory object: %v", err) return err } @@ -78,7 +90,6 @@ func Pack(filesToPack FilesToPack, outputFilename string) error { hdr.DirectoryOffset = packer.Pos() hdr.DirectoryLength = uint64(len(m)) if _, err := packer.Write(m); err != nil { - // TODO: decorate return err } @@ -113,7 +124,7 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e data, err := unix.Mmap(int(f.Fd()), 0, int(fi.Size()), unix.PROT_READ, unix.MAP_SHARED) if err != nil { - // TODO: decorate + err = fmt.Errorf("mmap %s: %v", fileToPack.Filename, err) return } defer unix.Munmap(data) @@ -129,8 +140,7 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e Offset: packer.Pos(), Length: uint64(len(data)), } - if _, err = packer.CopyFrom(f); err != nil { - // TODO: decorate + if _, err = packer.CopyFrom(f, fi); err != nil { return } info.Uncompressed = fileData @@ -147,11 +157,14 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e fileData = &packed.FileData{ Offset: packer.Pos(), } - fileData.Length, err = packOneGzip(packer, data) + fileData.Length, err = packOneGzip(packer, data, + info.Uncompressed.Length) if err != nil { return } - info.Gzip = fileData + if fileData.Length > 0 { + info.Gzip = fileData + } } // brotli compression @@ -162,11 +175,14 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e fileData = &packed.FileData{ Offset: packer.Pos(), } - fileData.Length, err = packOneBrotli(packer, fileToPack.Filename) + fileData.Length, err = packOneBrotli(packer, + fileToPack.Filename, info.Uncompressed.Length) if err != nil { return } - info.Brotli = fileData + if fileData.Length > 0 { + info.Brotli = fileData + } } return @@ -178,7 +194,8 @@ func etag(in []byte) string { return fmt.Sprintf(`"1--%x"`, h.Sum(nil)) } -func packOneGzip(packer *packWriter, data []byte) (uint64, error) { +func packOneGzip(packer *packWriter, data []byte, uncompressedSize uint64, +) (uint64, error) { // write via temporary file tmpfile, err := ioutil.TempFile("", "") if err != nil { @@ -202,10 +219,11 @@ func packOneGzip(packer *packWriter, data []byte) (uint64, error) { } // copy into packfile - return packer.CopyFrom(tmpfile) + return packer.CopyIfSaving(tmpfile, uncompressedSize) } -func packOneBrotli(packer *packWriter, filename string) (uint64, error) { +func packOneBrotli(packer *packWriter, filename string, uncompressedSize uint64, +) (uint64, error) { // write via temporary file tmpfile, err := ioutil.TempFile("", "") if err != nil { @@ -219,13 +237,12 @@ func packOneBrotli(packer *packWriter, filename string) (uint64, error) { "--output", tmpfile.Name()) out, err := cmd.CombinedOutput() if err != nil { - // TODO: decorate - _ = out + err = fmt.Errorf("brotli: %v (process reported: %s)", err, out) return 0, err } // copy into packfile - return packer.CopyFrom(tmpfile) + return packer.CopyIfSaving(tmpfile, uncompressedSize) } type packWriter struct { @@ -272,7 +289,7 @@ func (pw *packWriter) Pad() error { return pw.err } -func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) { +func (pw *packWriter) CopyIfSaving(in *os.File, uncompressedSize uint64) (uint64, error) { if pw.err != nil { return 0, pw.err } @@ -282,8 +299,22 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) { pw.err = err return 0, pw.err } + sz := uint64(fi.Size()) - fmt.Fprintf(os.Stderr, "[DEBUG] in size=%d\n", fi.Size()) + if sz+minCompressionSaving > uncompressedSize { + return 0, nil + } + if sz+(uncompressedSize>>minCompressionFraction) > uncompressedSize { + return 0, nil + } + + return pw.CopyFrom(in, fi) +} + +func (pw *packWriter) CopyFrom(in *os.File, fi os.FileInfo) (uint64, error) { + if pw.err != nil { + return 0, pw.err + } var off int64 remain := fi.Size() @@ -295,12 +326,11 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) { amt = int(remain) } - amt, err = unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt) - fmt.Fprintf(os.Stderr, "[DEBUG] sendfile=%d [off now %d]\n", amt, off) + amt, err := unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt) remain -= int64(amt) - //off += int64(amt) if err != nil { - pw.err = err + pw.err = fmt.Errorf("sendfile (copying data to "+ + "htpack): %v", err) return uint64(off), pw.err } }