Further work; tidying up, begin cmdline tool
This commit is contained in:
parent
2a25d80249
commit
5c05a75fba
|
@ -1,12 +1,35 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/lwithers/htpack/internal/packed"
|
"github.com/lwithers/htpack/internal/packed"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var inspectCmd = &cobra.Command{
|
||||||
|
Use: "inspect",
|
||||||
|
Short: "View contents of an htpack file",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New("must specify one or more files")
|
||||||
|
}
|
||||||
|
|
||||||
|
var exitCode int
|
||||||
|
for _, filename := range args {
|
||||||
|
if err := Inspect(filename); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: %v\n",
|
||||||
|
filename, err)
|
||||||
|
exitCode = 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
os.Exit(exitCode)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
// Inspect a packfile.
|
// Inspect a packfile.
|
||||||
// TODO: verify etag; verify integrity of compressed data.
|
// TODO: verify etag; verify integrity of compressed data.
|
||||||
// TODO: skip Gzip/Brotli if not present; print ratio.
|
// TODO: skip Gzip/Brotli if not present; print ratio.
|
||||||
|
|
|
@ -0,0 +1,28 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var rootCmd = &cobra.Command{
|
||||||
|
Use: "htpacker",
|
||||||
|
Short: "htpacker packs static files into a blob that can be served efficiently over HTTP",
|
||||||
|
Long: `Creates .htpack files comprising one or more static assets, and
|
||||||
|
compressed versions thereof. A YAML specification of files to pack may be
|
||||||
|
provided or generated on demand; or files and directories can be listed as
|
||||||
|
arguments.`,
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
rootCmd.AddCommand(packCmd)
|
||||||
|
//rootCmd.AddCommand(yamlCmd)
|
||||||
|
rootCmd.AddCommand(inspectCmd)
|
||||||
|
|
||||||
|
if err := rootCmd.Execute(); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,75 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/lwithers/htpack/packer"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
yaml "gopkg.in/yaml.v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
var packCmd = &cobra.Command{
|
||||||
|
Use: "pack",
|
||||||
|
Short: "creates a packfile from a YAML spec or set of files/dirs",
|
||||||
|
RunE: func(c *cobra.Command, args []string) error {
|
||||||
|
spec, err := c.Flags().GetString("spec")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if spec == "" {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New("need --yaml, " +
|
||||||
|
"or one or more filenames")
|
||||||
|
}
|
||||||
|
err = PackFiles(c, args)
|
||||||
|
} else {
|
||||||
|
if len(args) != 0 {
|
||||||
|
return errors.New("cannot specify files " +
|
||||||
|
"when using --yaml")
|
||||||
|
}
|
||||||
|
err = PackSpec(c, spec)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
packCmd.Flags().StringP("out", "O", "",
|
||||||
|
"Output filename")
|
||||||
|
packCmd.MarkFlagRequired("out")
|
||||||
|
packCmd.Flags().StringP("spec", "y", "",
|
||||||
|
"YAML specification file (if not present, just pack files)")
|
||||||
|
packCmd.Flags().StringP("chdir", "C", "",
|
||||||
|
"Change to directory before searching for input files")
|
||||||
|
}
|
||||||
|
|
||||||
|
func PackFiles(c *cobra.Command, args []string) error {
|
||||||
|
// TODO
|
||||||
|
return errors.New("not implemented yet")
|
||||||
|
}
|
||||||
|
|
||||||
|
func PackSpec(c *cobra.Command, spec string) error {
|
||||||
|
raw, err := ioutil.ReadFile(spec)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var ftp packer.FilesToPack
|
||||||
|
if err := yaml.UnmarshalStrict(raw, &ftp); err != nil {
|
||||||
|
return fmt.Errorf("parsing YAML spec %s: %v", spec, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: chdir
|
||||||
|
|
||||||
|
out, _ := c.Flags().GetString("out")
|
||||||
|
return packer.Pack(ftp, out)
|
||||||
|
}
|
|
@ -1,31 +0,0 @@
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
yaml "gopkg.in/yaml.v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
//if err := dopack(); err != nil {
|
|
||||||
if err := Inspect("out.htpack"); err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func dopack() error {
|
|
||||||
raw, err := ioutil.ReadFile("in.yaml")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var ftp FilesToPack
|
|
||||||
if err := yaml.UnmarshalStrict(raw, &ftp); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return Pack(ftp, "out.htpack")
|
|
||||||
}
|
|
|
@ -1,4 +1,4 @@
|
||||||
package main
|
package packer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
@ -16,8 +16,6 @@ import (
|
||||||
"github.com/lwithers/pkg/writefile"
|
"github.com/lwithers/pkg/writefile"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TODO: abandon packed version if no size saving
|
|
||||||
|
|
||||||
var BrotliPath string = "brotli"
|
var BrotliPath string = "brotli"
|
||||||
|
|
||||||
type FilesToPack map[string]FileToPack
|
type FilesToPack map[string]FileToPack
|
||||||
|
@ -37,6 +35,20 @@ type packInfo struct {
|
||||||
offset, len uint64
|
offset, len uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
// minCompressionSaving means we'll only use the compressed version of
|
||||||
|
// the file if it's at least this many bytes smaller than the original.
|
||||||
|
// Chosen somewhat arbitrarily; we have to add an HTTP header, and the
|
||||||
|
// decompression overhead is not zero.
|
||||||
|
minCompressionSaving = 128
|
||||||
|
|
||||||
|
// minCompressionFraction means we'll only use the compressed version of
|
||||||
|
// the file if it's at least (origSize>>minCompressionFraction) bytes
|
||||||
|
// smaller than the original. This is a guess at when the decompression
|
||||||
|
// overhead outweighs the time saved in transmission.
|
||||||
|
minCompressionFraction = 7 // i.e. files must be at least 1/128 smaller
|
||||||
|
)
|
||||||
|
|
||||||
// Pack a file.
|
// Pack a file.
|
||||||
func Pack(filesToPack FilesToPack, outputFilename string) error {
|
func Pack(filesToPack FilesToPack, outputFilename string) error {
|
||||||
finalFname, outputFile, err := writefile.New(outputFilename)
|
finalFname, outputFile, err := writefile.New(outputFilename)
|
||||||
|
@ -70,7 +82,7 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
|
||||||
|
|
||||||
// write the directory
|
// write the directory
|
||||||
if m, err = dir.Marshal(); err != nil {
|
if m, err = dir.Marshal(); err != nil {
|
||||||
// TODO: decorate
|
err = fmt.Errorf("marshaling directory object: %v", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -78,7 +90,6 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
|
||||||
hdr.DirectoryOffset = packer.Pos()
|
hdr.DirectoryOffset = packer.Pos()
|
||||||
hdr.DirectoryLength = uint64(len(m))
|
hdr.DirectoryLength = uint64(len(m))
|
||||||
if _, err := packer.Write(m); err != nil {
|
if _, err := packer.Write(m); err != nil {
|
||||||
// TODO: decorate
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -113,7 +124,7 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e
|
||||||
data, err := unix.Mmap(int(f.Fd()), 0, int(fi.Size()),
|
data, err := unix.Mmap(int(f.Fd()), 0, int(fi.Size()),
|
||||||
unix.PROT_READ, unix.MAP_SHARED)
|
unix.PROT_READ, unix.MAP_SHARED)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// TODO: decorate
|
err = fmt.Errorf("mmap %s: %v", fileToPack.Filename, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer unix.Munmap(data)
|
defer unix.Munmap(data)
|
||||||
|
@ -129,8 +140,7 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e
|
||||||
Offset: packer.Pos(),
|
Offset: packer.Pos(),
|
||||||
Length: uint64(len(data)),
|
Length: uint64(len(data)),
|
||||||
}
|
}
|
||||||
if _, err = packer.CopyFrom(f); err != nil {
|
if _, err = packer.CopyFrom(f, fi); err != nil {
|
||||||
// TODO: decorate
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
info.Uncompressed = fileData
|
info.Uncompressed = fileData
|
||||||
|
@ -147,11 +157,14 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e
|
||||||
fileData = &packed.FileData{
|
fileData = &packed.FileData{
|
||||||
Offset: packer.Pos(),
|
Offset: packer.Pos(),
|
||||||
}
|
}
|
||||||
fileData.Length, err = packOneGzip(packer, data)
|
fileData.Length, err = packOneGzip(packer, data,
|
||||||
|
info.Uncompressed.Length)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
info.Gzip = fileData
|
if fileData.Length > 0 {
|
||||||
|
info.Gzip = fileData
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// brotli compression
|
// brotli compression
|
||||||
|
@ -162,11 +175,14 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e
|
||||||
fileData = &packed.FileData{
|
fileData = &packed.FileData{
|
||||||
Offset: packer.Pos(),
|
Offset: packer.Pos(),
|
||||||
}
|
}
|
||||||
fileData.Length, err = packOneBrotli(packer, fileToPack.Filename)
|
fileData.Length, err = packOneBrotli(packer,
|
||||||
|
fileToPack.Filename, info.Uncompressed.Length)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
info.Brotli = fileData
|
if fileData.Length > 0 {
|
||||||
|
info.Brotli = fileData
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
|
@ -178,7 +194,8 @@ func etag(in []byte) string {
|
||||||
return fmt.Sprintf(`"1--%x"`, h.Sum(nil))
|
return fmt.Sprintf(`"1--%x"`, h.Sum(nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
func packOneGzip(packer *packWriter, data []byte) (uint64, error) {
|
func packOneGzip(packer *packWriter, data []byte, uncompressedSize uint64,
|
||||||
|
) (uint64, error) {
|
||||||
// write via temporary file
|
// write via temporary file
|
||||||
tmpfile, err := ioutil.TempFile("", "")
|
tmpfile, err := ioutil.TempFile("", "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -202,10 +219,11 @@ func packOneGzip(packer *packWriter, data []byte) (uint64, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// copy into packfile
|
// copy into packfile
|
||||||
return packer.CopyFrom(tmpfile)
|
return packer.CopyIfSaving(tmpfile, uncompressedSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
func packOneBrotli(packer *packWriter, filename string) (uint64, error) {
|
func packOneBrotli(packer *packWriter, filename string, uncompressedSize uint64,
|
||||||
|
) (uint64, error) {
|
||||||
// write via temporary file
|
// write via temporary file
|
||||||
tmpfile, err := ioutil.TempFile("", "")
|
tmpfile, err := ioutil.TempFile("", "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -219,13 +237,12 @@ func packOneBrotli(packer *packWriter, filename string) (uint64, error) {
|
||||||
"--output", tmpfile.Name())
|
"--output", tmpfile.Name())
|
||||||
out, err := cmd.CombinedOutput()
|
out, err := cmd.CombinedOutput()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// TODO: decorate
|
err = fmt.Errorf("brotli: %v (process reported: %s)", err, out)
|
||||||
_ = out
|
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// copy into packfile
|
// copy into packfile
|
||||||
return packer.CopyFrom(tmpfile)
|
return packer.CopyIfSaving(tmpfile, uncompressedSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
type packWriter struct {
|
type packWriter struct {
|
||||||
|
@ -272,7 +289,7 @@ func (pw *packWriter) Pad() error {
|
||||||
return pw.err
|
return pw.err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
|
func (pw *packWriter) CopyIfSaving(in *os.File, uncompressedSize uint64) (uint64, error) {
|
||||||
if pw.err != nil {
|
if pw.err != nil {
|
||||||
return 0, pw.err
|
return 0, pw.err
|
||||||
}
|
}
|
||||||
|
@ -282,8 +299,22 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
|
||||||
pw.err = err
|
pw.err = err
|
||||||
return 0, pw.err
|
return 0, pw.err
|
||||||
}
|
}
|
||||||
|
sz := uint64(fi.Size())
|
||||||
|
|
||||||
fmt.Fprintf(os.Stderr, "[DEBUG] in size=%d\n", fi.Size())
|
if sz+minCompressionSaving > uncompressedSize {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
if sz+(uncompressedSize>>minCompressionFraction) > uncompressedSize {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return pw.CopyFrom(in, fi)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (pw *packWriter) CopyFrom(in *os.File, fi os.FileInfo) (uint64, error) {
|
||||||
|
if pw.err != nil {
|
||||||
|
return 0, pw.err
|
||||||
|
}
|
||||||
|
|
||||||
var off int64
|
var off int64
|
||||||
remain := fi.Size()
|
remain := fi.Size()
|
||||||
|
@ -295,12 +326,11 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
|
||||||
amt = int(remain)
|
amt = int(remain)
|
||||||
}
|
}
|
||||||
|
|
||||||
amt, err = unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt)
|
amt, err := unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt)
|
||||||
fmt.Fprintf(os.Stderr, "[DEBUG] sendfile=%d [off now %d]\n", amt, off)
|
|
||||||
remain -= int64(amt)
|
remain -= int64(amt)
|
||||||
//off += int64(amt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
pw.err = err
|
pw.err = fmt.Errorf("sendfile (copying data to "+
|
||||||
|
"htpack): %v", err)
|
||||||
return uint64(off), pw.err
|
return uint64(off), pw.err
|
||||||
}
|
}
|
||||||
}
|
}
|
Loading…
Reference in New Issue