Further work; tidying up, begin cmdline tool
This commit is contained in:
Laurence Withers
parent
2a25d80249
commit
5c05a75fba
5 changed files with 180 additions and 55 deletions
|
|
@ -1,12 +1,35 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/lwithers/htpack/internal/packed"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var inspectCmd = &cobra.Command{
|
||||
Use: "inspect",
|
||||
Short: "View contents of an htpack file",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("must specify one or more files")
|
||||
}
|
||||
|
||||
var exitCode int
|
||||
for _, filename := range args {
|
||||
if err := Inspect(filename); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n",
|
||||
filename, err)
|
||||
exitCode = 1
|
||||
}
|
||||
}
|
||||
os.Exit(exitCode)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
// Inspect a packfile.
|
||||
// TODO: verify etag; verify integrity of compressed data.
|
||||
// TODO: skip Gzip/Brotli if not present; print ratio.
|
||||
|
|
|
|||
28
cmd/htpacker/main.go
Normal file
28
cmd/htpacker/main.go
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "htpacker",
|
||||
Short: "htpacker packs static files into a blob that can be served efficiently over HTTP",
|
||||
Long: `Creates .htpack files comprising one or more static assets, and
|
||||
compressed versions thereof. A YAML specification of files to pack may be
|
||||
provided or generated on demand; or files and directories can be listed as
|
||||
arguments.`,
|
||||
}
|
||||
|
||||
func main() {
|
||||
rootCmd.AddCommand(packCmd)
|
||||
//rootCmd.AddCommand(yamlCmd)
|
||||
rootCmd.AddCommand(inspectCmd)
|
||||
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
75
cmd/htpacker/pack.go
Normal file
75
cmd/htpacker/pack.go
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
||||
"github.com/lwithers/htpack/packer"
|
||||
"github.com/spf13/cobra"
|
||||
yaml "gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
var packCmd = &cobra.Command{
|
||||
Use: "pack",
|
||||
Short: "creates a packfile from a YAML spec or set of files/dirs",
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
spec, err := c.Flags().GetString("spec")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if spec == "" {
|
||||
if len(args) == 0 {
|
||||
return errors.New("need --yaml, " +
|
||||
"or one or more filenames")
|
||||
}
|
||||
err = PackFiles(c, args)
|
||||
} else {
|
||||
if len(args) != 0 {
|
||||
return errors.New("cannot specify files " +
|
||||
"when using --yaml")
|
||||
}
|
||||
err = PackSpec(c, spec)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
packCmd.Flags().StringP("out", "O", "",
|
||||
"Output filename")
|
||||
packCmd.MarkFlagRequired("out")
|
||||
packCmd.Flags().StringP("spec", "y", "",
|
||||
"YAML specification file (if not present, just pack files)")
|
||||
packCmd.Flags().StringP("chdir", "C", "",
|
||||
"Change to directory before searching for input files")
|
||||
}
|
||||
|
||||
func PackFiles(c *cobra.Command, args []string) error {
|
||||
// TODO
|
||||
return errors.New("not implemented yet")
|
||||
}
|
||||
|
||||
func PackSpec(c *cobra.Command, spec string) error {
|
||||
raw, err := ioutil.ReadFile(spec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var ftp packer.FilesToPack
|
||||
if err := yaml.UnmarshalStrict(raw, &ftp); err != nil {
|
||||
return fmt.Errorf("parsing YAML spec %s: %v", spec, err)
|
||||
}
|
||||
|
||||
// TODO: chdir
|
||||
|
||||
out, _ := c.Flags().GetString("out")
|
||||
return packer.Pack(ftp, out)
|
||||
}
|
||||
|
|
@ -1,309 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha512"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/foobaz/go-zopfli/zopfli"
|
||||
"github.com/lwithers/htpack/internal/packed"
|
||||
"github.com/lwithers/pkg/writefile"
|
||||
)
|
||||
|
||||
// TODO: abandon packed version if no size saving
|
||||
|
||||
var BrotliPath string = "brotli"
|
||||
|
||||
type FilesToPack map[string]FileToPack
|
||||
|
||||
type FileToPack struct {
|
||||
Filename string `yaml:"filename"`
|
||||
ContentType string `yaml:"content_type"`
|
||||
DisableCompression bool `yaml:"disable_compression"`
|
||||
DisableGzip bool `yaml:"disable_gzip"`
|
||||
DisableBrotli bool `yaml:"disable_brotli"`
|
||||
|
||||
uncompressed, gzip, brotli packInfo
|
||||
}
|
||||
|
||||
type packInfo struct {
|
||||
present bool
|
||||
offset, len uint64
|
||||
}
|
||||
|
||||
// Pack a file.
|
||||
func Pack(filesToPack FilesToPack, outputFilename string) error {
|
||||
finalFname, outputFile, err := writefile.New(outputFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer writefile.Abort(outputFile)
|
||||
packer := &packWriter{f: outputFile}
|
||||
|
||||
// write initial header (will rewrite offset/length when known)
|
||||
hdr := &packed.Header{
|
||||
Magic: packed.Magic,
|
||||
Version: packed.VersionInitial,
|
||||
DirectoryOffset: 1,
|
||||
DirectoryLength: 1,
|
||||
}
|
||||
m, _ := hdr.Marshal()
|
||||
packer.Write(m)
|
||||
|
||||
dir := packed.Directory{
|
||||
Files: make(map[string]*packed.File),
|
||||
}
|
||||
|
||||
for path, fileToPack := range filesToPack {
|
||||
info, err := packOne(packer, fileToPack)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir.Files[path] = &info
|
||||
}
|
||||
|
||||
// write the directory
|
||||
if m, err = dir.Marshal(); err != nil {
|
||||
// TODO: decorate
|
||||
return err
|
||||
}
|
||||
|
||||
packer.Pad()
|
||||
hdr.DirectoryOffset = packer.Pos()
|
||||
hdr.DirectoryLength = uint64(len(m))
|
||||
if _, err := packer.Write(m); err != nil {
|
||||
// TODO: decorate
|
||||
return err
|
||||
}
|
||||
|
||||
// write header at start of file
|
||||
m, _ = hdr.Marshal()
|
||||
if _, err = outputFile.WriteAt(m, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// all done!
|
||||
return writefile.Commit(finalFname, outputFile)
|
||||
}
|
||||
|
||||
func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err error) {
|
||||
// implementation detail: write files at a page boundary
|
||||
if err = packer.Pad(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// open and mmap input file
|
||||
f, err := os.Open(fileToPack.Filename)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
data, err := unix.Mmap(int(f.Fd()), 0, int(fi.Size()),
|
||||
unix.PROT_READ, unix.MAP_SHARED)
|
||||
if err != nil {
|
||||
// TODO: decorate
|
||||
return
|
||||
}
|
||||
defer unix.Munmap(data)
|
||||
|
||||
info.Etag = etag(data)
|
||||
info.ContentType = fileToPack.ContentType
|
||||
if info.ContentType == "" {
|
||||
info.ContentType = http.DetectContentType(data)
|
||||
}
|
||||
|
||||
// copy the uncompressed version
|
||||
fileData := &packed.FileData{
|
||||
Offset: packer.Pos(),
|
||||
Length: uint64(len(data)),
|
||||
}
|
||||
if _, err = packer.CopyFrom(f); err != nil {
|
||||
// TODO: decorate
|
||||
return
|
||||
}
|
||||
info.Uncompressed = fileData
|
||||
|
||||
if fileToPack.DisableCompression {
|
||||
return
|
||||
}
|
||||
|
||||
// gzip compression
|
||||
if !fileToPack.DisableGzip {
|
||||
if err = packer.Pad(); err != nil {
|
||||
return
|
||||
}
|
||||
fileData = &packed.FileData{
|
||||
Offset: packer.Pos(),
|
||||
}
|
||||
fileData.Length, err = packOneGzip(packer, data)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
info.Gzip = fileData
|
||||
}
|
||||
|
||||
// brotli compression
|
||||
if BrotliPath != "" && !fileToPack.DisableBrotli {
|
||||
if err = packer.Pad(); err != nil {
|
||||
return
|
||||
}
|
||||
fileData = &packed.FileData{
|
||||
Offset: packer.Pos(),
|
||||
}
|
||||
fileData.Length, err = packOneBrotli(packer, fileToPack.Filename)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
info.Brotli = fileData
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func etag(in []byte) string {
|
||||
h := sha512.New384()
|
||||
h.Write(in)
|
||||
return fmt.Sprintf(`"1--%x"`, h.Sum(nil))
|
||||
}
|
||||
|
||||
func packOneGzip(packer *packWriter, data []byte) (uint64, error) {
|
||||
// write via temporary file
|
||||
tmpfile, err := ioutil.TempFile("", "")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer os.Remove(tmpfile.Name())
|
||||
defer tmpfile.Close()
|
||||
|
||||
// compress
|
||||
opts := zopfli.DefaultOptions()
|
||||
if len(data) > (10 << 20) { // 10MiB
|
||||
opts.NumIterations = 5
|
||||
}
|
||||
|
||||
buf := bufio.NewWriter(tmpfile)
|
||||
if err = zopfli.GzipCompress(&opts, data, buf); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err = buf.Flush(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// copy into packfile
|
||||
return packer.CopyFrom(tmpfile)
|
||||
}
|
||||
|
||||
func packOneBrotli(packer *packWriter, filename string) (uint64, error) {
|
||||
// write via temporary file
|
||||
tmpfile, err := ioutil.TempFile("", "")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer os.Remove(tmpfile.Name())
|
||||
defer tmpfile.Close()
|
||||
|
||||
// compress via commandline
|
||||
cmd := exec.Command(BrotliPath, "--input", filename,
|
||||
"--output", tmpfile.Name())
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
// TODO: decorate
|
||||
_ = out
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// copy into packfile
|
||||
return packer.CopyFrom(tmpfile)
|
||||
}
|
||||
|
||||
type packWriter struct {
|
||||
f *os.File
|
||||
err error
|
||||
}
|
||||
|
||||
func (pw *packWriter) Write(buf []byte) (int, error) {
|
||||
if pw.err != nil {
|
||||
return 0, pw.err
|
||||
}
|
||||
n, err := pw.f.Write(buf)
|
||||
pw.err = err
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (pw *packWriter) Pos() uint64 {
|
||||
pos, err := pw.f.Seek(0, os.SEEK_CUR)
|
||||
if err != nil {
|
||||
pw.err = err
|
||||
}
|
||||
return uint64(pos)
|
||||
}
|
||||
|
||||
func (pw *packWriter) Pad() error {
|
||||
if pw.err != nil {
|
||||
return pw.err
|
||||
}
|
||||
|
||||
pos, err := pw.f.Seek(0, os.SEEK_CUR)
|
||||
if err != nil {
|
||||
pw.err = err
|
||||
return pw.err
|
||||
}
|
||||
|
||||
pos &= 0xFFF
|
||||
if pos == 0 {
|
||||
return pw.err
|
||||
}
|
||||
|
||||
if _, err = pw.f.Seek(4096-pos, os.SEEK_CUR); err != nil {
|
||||
pw.err = err
|
||||
}
|
||||
return pw.err
|
||||
}
|
||||
|
||||
func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
|
||||
if pw.err != nil {
|
||||
return 0, pw.err
|
||||
}
|
||||
|
||||
fi, err := in.Stat()
|
||||
if err != nil {
|
||||
pw.err = err
|
||||
return 0, pw.err
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "[DEBUG] in size=%d\n", fi.Size())
|
||||
|
||||
var off int64
|
||||
remain := fi.Size()
|
||||
for remain > 0 {
|
||||
var amt int
|
||||
if remain > (1 << 30) {
|
||||
amt = (1 << 30)
|
||||
} else {
|
||||
amt = int(remain)
|
||||
}
|
||||
|
||||
amt, err = unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt)
|
||||
fmt.Fprintf(os.Stderr, "[DEBUG] sendfile=%d [off now %d]\n", amt, off)
|
||||
remain -= int64(amt)
|
||||
//off += int64(amt)
|
||||
if err != nil {
|
||||
pw.err = err
|
||||
return uint64(off), pw.err
|
||||
}
|
||||
}
|
||||
|
||||
return uint64(off), nil
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
||||
yaml "gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
func main() {
|
||||
//if err := dopack(); err != nil {
|
||||
if err := Inspect("out.htpack"); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func dopack() error {
|
||||
raw, err := ioutil.ReadFile("in.yaml")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var ftp FilesToPack
|
||||
if err := yaml.UnmarshalStrict(raw, &ftp); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return Pack(ftp, "out.htpack")
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue