Further work-in-progress, basic packing and inspecting
This commit is contained in:
parent
e864d0829e
commit
d836bf3cc7
|
|
@ -1,7 +1,7 @@
|
|||
# HTTP resource pack server
|
||||
|
||||
A common scenario is that you have a set of static resources that you want to
|
||||
server up quickly via HTTP (for example: stylesheets, WASM).
|
||||
serve up quickly via HTTP (for example: stylesheets, WASM).
|
||||
|
||||
This package provides a `net/http`-compatible `http.Handler` to do so, with
|
||||
support for:
|
||||
|
|
@ -20,4 +20,3 @@ The workflow is as follows:
|
|||
Only the minimal header processing necessary for correctness (Content-Length,
|
||||
etc.) is carried out by `htpack.Handler`; the handler can be combined with
|
||||
middleware for further processing (adding headers, `http.StripPrefix`, etc.).
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/lwithers/htpack/internal/packed"
|
||||
)
|
||||
|
||||
// Inspect a packfile.
|
||||
// TODO: verify etag; verify integrity of compressed data.
|
||||
// TODO: skip Gzip/Brotli if not present; print ratio.
|
||||
func Inspect(filename string) error {
|
||||
f, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
hdr, dir, err := packed.Load(f)
|
||||
if hdr != nil {
|
||||
fmt.Printf("Header: %#v\n", hdr)
|
||||
}
|
||||
if dir != nil {
|
||||
fmt.Printf("%d files:\n", len(dir.Files))
|
||||
for path, info := range dir.Files {
|
||||
fmt.Printf(" • %s\n"+
|
||||
" · Etag: %s\n"+
|
||||
" · Content type: %s\n"+
|
||||
" · Uncompressed: %s (offset %d)\n"+
|
||||
" · Gzipped: %s (offset %d)\n"+
|
||||
" · Brotli: %s (offset %d)\n",
|
||||
path, info.Etag, info.ContentType,
|
||||
printSize(info.Uncompressed.Length), info.Uncompressed.Offset,
|
||||
printSize(info.Gzip.Length), info.Gzip.Offset,
|
||||
printSize(info.Brotli.Length), info.Brotli.Offset,
|
||||
)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func printSize(size uint64) string {
|
||||
switch {
|
||||
case size < 1<<10:
|
||||
return fmt.Sprintf("%d bytes", size)
|
||||
case size < 1<<15:
|
||||
return fmt.Sprintf("%.2f KiB", float64(size)/(1<<10))
|
||||
case size < 1<<20:
|
||||
return fmt.Sprintf("%.1f KiB", float64(size)/(1<<10))
|
||||
case size < 1<<25:
|
||||
return fmt.Sprintf("%.2f MiB", float64(size)/(1<<20))
|
||||
case size < 1<<30:
|
||||
return fmt.Sprintf("%.1f MiB", float64(size)/(1<<20))
|
||||
case size < 1<<35:
|
||||
return fmt.Sprintf("%.2f GiB", float64(size)/(1<<30))
|
||||
default:
|
||||
return fmt.Sprintf("%.1f GiB", float64(size)/(1<<30))
|
||||
}
|
||||
}
|
||||
|
|
@ -37,6 +37,7 @@ type packInfo struct {
|
|||
offset, len uint64
|
||||
}
|
||||
|
||||
// Pack a file.
|
||||
func Pack(filesToPack FilesToPack, outputFilename string) error {
|
||||
finalFname, outputFile, err := writefile.New(outputFilename)
|
||||
if err != nil {
|
||||
|
|
@ -47,8 +48,8 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
|
|||
|
||||
// write initial header (will rewrite offset/length when known)
|
||||
hdr := &packed.Header{
|
||||
Magic: 123,
|
||||
Version: 1,
|
||||
Magic: packed.Magic,
|
||||
Version: packed.VersionInitial,
|
||||
DirectoryOffset: 1,
|
||||
DirectoryLength: 1,
|
||||
}
|
||||
|
|
@ -117,7 +118,6 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e
|
|||
}
|
||||
defer unix.Munmap(data)
|
||||
|
||||
// TODO: content-type, etag
|
||||
info.Etag = etag(data)
|
||||
info.ContentType = fileToPack.ContentType
|
||||
if info.ContentType == "" {
|
||||
|
|
@ -283,6 +283,8 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
|
|||
return 0, pw.err
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "[DEBUG] in size=%d\n", fi.Size())
|
||||
|
||||
var off int64
|
||||
remain := fi.Size()
|
||||
for remain > 0 {
|
||||
|
|
@ -294,8 +296,9 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
|
|||
}
|
||||
|
||||
amt, err = unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt)
|
||||
fmt.Fprintf(os.Stderr, "[DEBUG] sendfile=%d [off now %d]\n", amt, off)
|
||||
remain -= int64(amt)
|
||||
off += int64(amt)
|
||||
//off += int64(amt)
|
||||
if err != nil {
|
||||
pw.err = err
|
||||
return uint64(off), pw.err
|
||||
|
|
|
|||
|
|
@ -9,13 +9,14 @@ import (
|
|||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
//if err := dopack(); err != nil {
|
||||
if err := Inspect("out.htpack"); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
func dopack() error {
|
||||
raw, err := ioutil.ReadFile("in.yaml")
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -0,0 +1,304 @@
|
|||
package packed
|
||||
|
||||
import (
|
||||
fmt "fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
// Magic number to identify .htpack files.
|
||||
Magic = 0xb6e61a4b415ed33b
|
||||
|
||||
// VersionInitial is the version number used by the initial packed
|
||||
// format. Loading a file with a higher version number will cause an
|
||||
// error to be returned.
|
||||
VersionInitial = 1
|
||||
)
|
||||
|
||||
// Load a ready-packed file.
|
||||
func Load(f *os.File) (*Header, *Directory, error) {
|
||||
hdr, err := loadHeader(f)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
dir, err := loadDirectory(f, hdr)
|
||||
if le, ok := err.(*LoadError); ok {
|
||||
// augment error
|
||||
le.Magic = hdr.Magic
|
||||
le.Version = hdr.Version
|
||||
}
|
||||
return hdr, dir, err // we may have a partial dir
|
||||
}
|
||||
|
||||
// loadHeader retrieves and decodes the header from the start of the file. It
|
||||
// ensures the magic number and the version number match. Errors are returned
|
||||
// as type LoadError.
|
||||
func loadHeader(f *os.File) (*Header, error) {
|
||||
raw := make([]byte, 36)
|
||||
if _, err := f.ReadAt(raw, 0); err != nil {
|
||||
return nil, &LoadError{
|
||||
Cause: IOError,
|
||||
Underlying: err,
|
||||
}
|
||||
}
|
||||
|
||||
hdr := new(Header)
|
||||
if err := hdr.Unmarshal(raw); err != nil {
|
||||
return nil, &LoadError{
|
||||
Cause: HeaderUnmarshalError,
|
||||
Underlying: err,
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case hdr.Magic != Magic:
|
||||
return nil, &LoadError{
|
||||
Cause: MagicMismatch,
|
||||
Magic: hdr.Magic,
|
||||
Version: hdr.Version,
|
||||
}
|
||||
case hdr.Version < VersionInitial:
|
||||
return nil, &LoadError{
|
||||
Cause: VersionTooOld,
|
||||
Magic: hdr.Magic,
|
||||
Version: hdr.Version,
|
||||
}
|
||||
case hdr.Version > VersionInitial:
|
||||
return nil, &LoadError{
|
||||
Cause: VersionTooNew,
|
||||
Magic: hdr.Magic,
|
||||
Version: hdr.Version,
|
||||
}
|
||||
}
|
||||
|
||||
return hdr, nil
|
||||
}
|
||||
|
||||
// loadDirectory reads the directory from a file. The directory is checked
|
||||
// for consistency (offsets, filenames) but not integrity (file data is not
|
||||
// read/checksummed).
|
||||
func loadDirectory(f *os.File, hdr *Header) (*Directory, error) {
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, &LoadError{
|
||||
Cause: IOError,
|
||||
Underlying: err,
|
||||
}
|
||||
}
|
||||
fileSize := uint64(fi.Size())
|
||||
|
||||
if hdr.DirectoryOffset+hdr.DirectoryLength > fileSize {
|
||||
return nil, &LoadError{
|
||||
Cause: BadOffsetError,
|
||||
}
|
||||
}
|
||||
|
||||
raw := make([]byte, hdr.DirectoryLength)
|
||||
if _, err := f.ReadAt(raw, int64(hdr.DirectoryOffset)); err != nil {
|
||||
return nil, &LoadError{
|
||||
Cause: IOError,
|
||||
Underlying: err,
|
||||
}
|
||||
}
|
||||
|
||||
dir := new(Directory)
|
||||
if err := dir.Unmarshal(raw); err != nil {
|
||||
return nil, &LoadError{
|
||||
Cause: DirectoryUnmarshalError,
|
||||
Underlying: err,
|
||||
}
|
||||
}
|
||||
|
||||
return dir, checkDirectory(dir, fileSize)
|
||||
}
|
||||
|
||||
// checkDirectory verifies the consistency of the htpack file (offsets,
|
||||
// filenames). It does not verify integrity (checksums).
|
||||
func checkDirectory(dir *Directory, fileSize uint64) error {
|
||||
files := map[string]struct{}{}
|
||||
|
||||
for filename, info := range dir.Files {
|
||||
var err error
|
||||
|
||||
// validate filename (not duplicate, canonical, etc)
|
||||
if _, dup := files[filename]; dup {
|
||||
err = fmt.Errorf("duplicate path %q", filename)
|
||||
}
|
||||
files[filename] = struct{}{}
|
||||
|
||||
if !path.IsAbs(filename) {
|
||||
err = fmt.Errorf("relative path %q", filename)
|
||||
}
|
||||
if path.Clean(filename) != filename {
|
||||
err = fmt.Errorf("non-canonical path %q", filename)
|
||||
}
|
||||
if err != nil {
|
||||
return &LoadError{
|
||||
Cause: InvalidPath,
|
||||
Underlying: err,
|
||||
Path: filename,
|
||||
}
|
||||
}
|
||||
|
||||
// ensure uncompressed data is present
|
||||
if info.Uncompressed == nil {
|
||||
return &LoadError{
|
||||
Cause: MissingUncompressed,
|
||||
Path: filename,
|
||||
}
|
||||
}
|
||||
|
||||
// validate offsets
|
||||
checkOffset(&err, filename, info.Uncompressed, fileSize)
|
||||
checkOffset(&err, filename, info.Gzip, fileSize)
|
||||
checkOffset(&err, filename, info.Brotli, fileSize)
|
||||
if err != nil {
|
||||
return &LoadError{
|
||||
Cause: BadOffsetError,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkOffset(perr *error, filename string, data *FileData, fileSize uint64) {
|
||||
if *perr != nil || data == nil {
|
||||
return
|
||||
}
|
||||
if data.Offset+data.Length > fileSize {
|
||||
*perr = &LoadError{
|
||||
Cause: BadOffsetError,
|
||||
Path: filename,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// LoadError reports a problem interpreting the header of a pack file.
|
||||
type LoadError struct {
|
||||
// Cause of the error.
|
||||
Cause ErrorCause
|
||||
|
||||
// Underlying may be set if there is some more information about the
|
||||
// error (e.g. I/O error).
|
||||
Underlying error
|
||||
|
||||
// Magic as read from the file.
|
||||
Magic uint64
|
||||
|
||||
// Version as read from the file.
|
||||
Version uint64
|
||||
|
||||
// Path of an individual file within the pack, if relevant.
|
||||
Path string
|
||||
}
|
||||
|
||||
// ErrorCause enumerates the possible reasons for failure.
|
||||
type ErrorCause int
|
||||
|
||||
const (
|
||||
// HeaderUnmarshalError means we could not unmarshal the protobuf object
|
||||
// at the head of the file.
|
||||
HeaderUnmarshalError ErrorCause = iota
|
||||
|
||||
// DirectoryUnmarshalError means we could not unmarshal the protobuf
|
||||
// object holding the directory contents.
|
||||
DirectoryUnmarshalError
|
||||
|
||||
// IOError indicates we could not read a protobuf object.
|
||||
IOError
|
||||
|
||||
// MagicMismatch occurs if the recorded magic number does not match
|
||||
// the well-known constant Magic.
|
||||
MagicMismatch
|
||||
|
||||
// VersionTooNew indicates that the file has a version number ahead of
|
||||
// what this package can parse.
|
||||
VersionTooNew
|
||||
|
||||
// VersionTooOld indicates that the file has a version number older than
|
||||
// thisk package can parse.
|
||||
VersionTooOld
|
||||
|
||||
// BadOffsetError means that the header or directory has indicated a
|
||||
// position that lies outside of the file.
|
||||
BadOffsetError
|
||||
|
||||
// InvalidPath is returned for a duplicate or otherwise invalid path.
|
||||
// Underlying is set to a free-form string error describing the path.
|
||||
InvalidPath
|
||||
|
||||
// MissingUncompressed indicates that a file in the pack does not have
|
||||
// an uncompressed version present, which is mandatory.
|
||||
MissingUncompressed
|
||||
)
|
||||
|
||||
// Desc returns a description of the error cause.
|
||||
func (le *LoadError) Desc() string {
|
||||
switch le.Cause {
|
||||
case HeaderUnmarshalError:
|
||||
return "not a .htpack file (header not valid packed.Header)"
|
||||
case DirectoryUnmarshalError:
|
||||
return "file corrupt (directory not valid packed.Directory)"
|
||||
case IOError:
|
||||
return "error reading from file"
|
||||
case MagicMismatch:
|
||||
return "magic number does not match"
|
||||
case VersionTooNew:
|
||||
return "version too new"
|
||||
case VersionTooOld:
|
||||
return "version too old"
|
||||
case BadOffsetError:
|
||||
return "file corrupt/truncated (offset past end of file)"
|
||||
case InvalidPath:
|
||||
return "filename invalid"
|
||||
case MissingUncompressed:
|
||||
return "missing uncompressed version"
|
||||
default:
|
||||
return "unknown error"
|
||||
}
|
||||
}
|
||||
|
||||
// Error returns a concise description of the error.
|
||||
func (le *LoadError) Error() string {
|
||||
var b strings.Builder
|
||||
|
||||
b.WriteString(le.Desc())
|
||||
|
||||
var underlying, magic, version, path bool
|
||||
switch le.Cause {
|
||||
case HeaderUnmarshalError, IOError:
|
||||
underlying = true
|
||||
case MagicMismatch:
|
||||
magic = true
|
||||
case VersionTooNew, VersionTooOld:
|
||||
version = true
|
||||
case BadOffsetError:
|
||||
path = le.Path != ""
|
||||
case InvalidPath, MissingUncompressed:
|
||||
path = true
|
||||
}
|
||||
|
||||
if underlying {
|
||||
b.WriteString(" [")
|
||||
b.WriteString(le.Underlying.Error())
|
||||
b.WriteString("]")
|
||||
}
|
||||
if magic {
|
||||
fmt.Fprintf(&b, " (found magic 0x%X, expected 0x%X)",
|
||||
le.Magic, uint64(Magic))
|
||||
}
|
||||
if version {
|
||||
fmt.Fprintf(&b, " (found version %d; oldest supported: "+
|
||||
"%d, newest: %d",
|
||||
le.Version, VersionInitial, VersionInitial)
|
||||
}
|
||||
if path {
|
||||
fmt.Fprintf(&b, " (path %q)", le.Path)
|
||||
}
|
||||
|
||||
return b.String()
|
||||
}
|
||||
Loading…
Reference in New Issue