Further work-in-progress, basic packing and inspecting
This commit is contained in:
parent
e864d0829e
commit
d836bf3cc7
|
@ -1,7 +1,7 @@
|
||||||
# HTTP resource pack server
|
# HTTP resource pack server
|
||||||
|
|
||||||
A common scenario is that you have a set of static resources that you want to
|
A common scenario is that you have a set of static resources that you want to
|
||||||
server up quickly via HTTP (for example: stylesheets, WASM).
|
serve up quickly via HTTP (for example: stylesheets, WASM).
|
||||||
|
|
||||||
This package provides a `net/http`-compatible `http.Handler` to do so, with
|
This package provides a `net/http`-compatible `http.Handler` to do so, with
|
||||||
support for:
|
support for:
|
||||||
|
@ -20,4 +20,3 @@ The workflow is as follows:
|
||||||
Only the minimal header processing necessary for correctness (Content-Length,
|
Only the minimal header processing necessary for correctness (Content-Length,
|
||||||
etc.) is carried out by `htpack.Handler`; the handler can be combined with
|
etc.) is carried out by `htpack.Handler`; the handler can be combined with
|
||||||
middleware for further processing (adding headers, `http.StripPrefix`, etc.).
|
middleware for further processing (adding headers, `http.StripPrefix`, etc.).
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,60 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/lwithers/htpack/internal/packed"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Inspect a packfile.
|
||||||
|
// TODO: verify etag; verify integrity of compressed data.
|
||||||
|
// TODO: skip Gzip/Brotli if not present; print ratio.
|
||||||
|
func Inspect(filename string) error {
|
||||||
|
f, err := os.Open(filename)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
hdr, dir, err := packed.Load(f)
|
||||||
|
if hdr != nil {
|
||||||
|
fmt.Printf("Header: %#v\n", hdr)
|
||||||
|
}
|
||||||
|
if dir != nil {
|
||||||
|
fmt.Printf("%d files:\n", len(dir.Files))
|
||||||
|
for path, info := range dir.Files {
|
||||||
|
fmt.Printf(" • %s\n"+
|
||||||
|
" · Etag: %s\n"+
|
||||||
|
" · Content type: %s\n"+
|
||||||
|
" · Uncompressed: %s (offset %d)\n"+
|
||||||
|
" · Gzipped: %s (offset %d)\n"+
|
||||||
|
" · Brotli: %s (offset %d)\n",
|
||||||
|
path, info.Etag, info.ContentType,
|
||||||
|
printSize(info.Uncompressed.Length), info.Uncompressed.Offset,
|
||||||
|
printSize(info.Gzip.Length), info.Gzip.Offset,
|
||||||
|
printSize(info.Brotli.Length), info.Brotli.Offset,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func printSize(size uint64) string {
|
||||||
|
switch {
|
||||||
|
case size < 1<<10:
|
||||||
|
return fmt.Sprintf("%d bytes", size)
|
||||||
|
case size < 1<<15:
|
||||||
|
return fmt.Sprintf("%.2f KiB", float64(size)/(1<<10))
|
||||||
|
case size < 1<<20:
|
||||||
|
return fmt.Sprintf("%.1f KiB", float64(size)/(1<<10))
|
||||||
|
case size < 1<<25:
|
||||||
|
return fmt.Sprintf("%.2f MiB", float64(size)/(1<<20))
|
||||||
|
case size < 1<<30:
|
||||||
|
return fmt.Sprintf("%.1f MiB", float64(size)/(1<<20))
|
||||||
|
case size < 1<<35:
|
||||||
|
return fmt.Sprintf("%.2f GiB", float64(size)/(1<<30))
|
||||||
|
default:
|
||||||
|
return fmt.Sprintf("%.1f GiB", float64(size)/(1<<30))
|
||||||
|
}
|
||||||
|
}
|
|
@ -37,6 +37,7 @@ type packInfo struct {
|
||||||
offset, len uint64
|
offset, len uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Pack a file.
|
||||||
func Pack(filesToPack FilesToPack, outputFilename string) error {
|
func Pack(filesToPack FilesToPack, outputFilename string) error {
|
||||||
finalFname, outputFile, err := writefile.New(outputFilename)
|
finalFname, outputFile, err := writefile.New(outputFilename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -47,8 +48,8 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
|
||||||
|
|
||||||
// write initial header (will rewrite offset/length when known)
|
// write initial header (will rewrite offset/length when known)
|
||||||
hdr := &packed.Header{
|
hdr := &packed.Header{
|
||||||
Magic: 123,
|
Magic: packed.Magic,
|
||||||
Version: 1,
|
Version: packed.VersionInitial,
|
||||||
DirectoryOffset: 1,
|
DirectoryOffset: 1,
|
||||||
DirectoryLength: 1,
|
DirectoryLength: 1,
|
||||||
}
|
}
|
||||||
|
@ -117,7 +118,6 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e
|
||||||
}
|
}
|
||||||
defer unix.Munmap(data)
|
defer unix.Munmap(data)
|
||||||
|
|
||||||
// TODO: content-type, etag
|
|
||||||
info.Etag = etag(data)
|
info.Etag = etag(data)
|
||||||
info.ContentType = fileToPack.ContentType
|
info.ContentType = fileToPack.ContentType
|
||||||
if info.ContentType == "" {
|
if info.ContentType == "" {
|
||||||
|
@ -283,6 +283,8 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
|
||||||
return 0, pw.err
|
return 0, pw.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(os.Stderr, "[DEBUG] in size=%d\n", fi.Size())
|
||||||
|
|
||||||
var off int64
|
var off int64
|
||||||
remain := fi.Size()
|
remain := fi.Size()
|
||||||
for remain > 0 {
|
for remain > 0 {
|
||||||
|
@ -294,8 +296,9 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
amt, err = unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt)
|
amt, err = unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt)
|
||||||
|
fmt.Fprintf(os.Stderr, "[DEBUG] sendfile=%d [off now %d]\n", amt, off)
|
||||||
remain -= int64(amt)
|
remain -= int64(amt)
|
||||||
off += int64(amt)
|
//off += int64(amt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
pw.err = err
|
pw.err = err
|
||||||
return uint64(off), pw.err
|
return uint64(off), pw.err
|
||||||
|
|
|
@ -9,13 +9,14 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
if err := run(); err != nil {
|
//if err := dopack(); err != nil {
|
||||||
|
if err := Inspect("out.htpack"); err != nil {
|
||||||
fmt.Fprintln(os.Stderr, err)
|
fmt.Fprintln(os.Stderr, err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func run() error {
|
func dopack() error {
|
||||||
raw, err := ioutil.ReadFile("in.yaml")
|
raw, err := ioutil.ReadFile("in.yaml")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
|
@ -0,0 +1,304 @@
|
||||||
|
package packed
|
||||||
|
|
||||||
|
import (
|
||||||
|
fmt "fmt"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// Magic number to identify .htpack files.
|
||||||
|
Magic = 0xb6e61a4b415ed33b
|
||||||
|
|
||||||
|
// VersionInitial is the version number used by the initial packed
|
||||||
|
// format. Loading a file with a higher version number will cause an
|
||||||
|
// error to be returned.
|
||||||
|
VersionInitial = 1
|
||||||
|
)
|
||||||
|
|
||||||
|
// Load a ready-packed file.
|
||||||
|
func Load(f *os.File) (*Header, *Directory, error) {
|
||||||
|
hdr, err := loadHeader(f)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
dir, err := loadDirectory(f, hdr)
|
||||||
|
if le, ok := err.(*LoadError); ok {
|
||||||
|
// augment error
|
||||||
|
le.Magic = hdr.Magic
|
||||||
|
le.Version = hdr.Version
|
||||||
|
}
|
||||||
|
return hdr, dir, err // we may have a partial dir
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadHeader retrieves and decodes the header from the start of the file. It
|
||||||
|
// ensures the magic number and the version number match. Errors are returned
|
||||||
|
// as type LoadError.
|
||||||
|
func loadHeader(f *os.File) (*Header, error) {
|
||||||
|
raw := make([]byte, 36)
|
||||||
|
if _, err := f.ReadAt(raw, 0); err != nil {
|
||||||
|
return nil, &LoadError{
|
||||||
|
Cause: IOError,
|
||||||
|
Underlying: err,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
hdr := new(Header)
|
||||||
|
if err := hdr.Unmarshal(raw); err != nil {
|
||||||
|
return nil, &LoadError{
|
||||||
|
Cause: HeaderUnmarshalError,
|
||||||
|
Underlying: err,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case hdr.Magic != Magic:
|
||||||
|
return nil, &LoadError{
|
||||||
|
Cause: MagicMismatch,
|
||||||
|
Magic: hdr.Magic,
|
||||||
|
Version: hdr.Version,
|
||||||
|
}
|
||||||
|
case hdr.Version < VersionInitial:
|
||||||
|
return nil, &LoadError{
|
||||||
|
Cause: VersionTooOld,
|
||||||
|
Magic: hdr.Magic,
|
||||||
|
Version: hdr.Version,
|
||||||
|
}
|
||||||
|
case hdr.Version > VersionInitial:
|
||||||
|
return nil, &LoadError{
|
||||||
|
Cause: VersionTooNew,
|
||||||
|
Magic: hdr.Magic,
|
||||||
|
Version: hdr.Version,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return hdr, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadDirectory reads the directory from a file. The directory is checked
|
||||||
|
// for consistency (offsets, filenames) but not integrity (file data is not
|
||||||
|
// read/checksummed).
|
||||||
|
func loadDirectory(f *os.File, hdr *Header) (*Directory, error) {
|
||||||
|
fi, err := f.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return nil, &LoadError{
|
||||||
|
Cause: IOError,
|
||||||
|
Underlying: err,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fileSize := uint64(fi.Size())
|
||||||
|
|
||||||
|
if hdr.DirectoryOffset+hdr.DirectoryLength > fileSize {
|
||||||
|
return nil, &LoadError{
|
||||||
|
Cause: BadOffsetError,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
raw := make([]byte, hdr.DirectoryLength)
|
||||||
|
if _, err := f.ReadAt(raw, int64(hdr.DirectoryOffset)); err != nil {
|
||||||
|
return nil, &LoadError{
|
||||||
|
Cause: IOError,
|
||||||
|
Underlying: err,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := new(Directory)
|
||||||
|
if err := dir.Unmarshal(raw); err != nil {
|
||||||
|
return nil, &LoadError{
|
||||||
|
Cause: DirectoryUnmarshalError,
|
||||||
|
Underlying: err,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return dir, checkDirectory(dir, fileSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkDirectory verifies the consistency of the htpack file (offsets,
|
||||||
|
// filenames). It does not verify integrity (checksums).
|
||||||
|
func checkDirectory(dir *Directory, fileSize uint64) error {
|
||||||
|
files := map[string]struct{}{}
|
||||||
|
|
||||||
|
for filename, info := range dir.Files {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
// validate filename (not duplicate, canonical, etc)
|
||||||
|
if _, dup := files[filename]; dup {
|
||||||
|
err = fmt.Errorf("duplicate path %q", filename)
|
||||||
|
}
|
||||||
|
files[filename] = struct{}{}
|
||||||
|
|
||||||
|
if !path.IsAbs(filename) {
|
||||||
|
err = fmt.Errorf("relative path %q", filename)
|
||||||
|
}
|
||||||
|
if path.Clean(filename) != filename {
|
||||||
|
err = fmt.Errorf("non-canonical path %q", filename)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return &LoadError{
|
||||||
|
Cause: InvalidPath,
|
||||||
|
Underlying: err,
|
||||||
|
Path: filename,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensure uncompressed data is present
|
||||||
|
if info.Uncompressed == nil {
|
||||||
|
return &LoadError{
|
||||||
|
Cause: MissingUncompressed,
|
||||||
|
Path: filename,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// validate offsets
|
||||||
|
checkOffset(&err, filename, info.Uncompressed, fileSize)
|
||||||
|
checkOffset(&err, filename, info.Gzip, fileSize)
|
||||||
|
checkOffset(&err, filename, info.Brotli, fileSize)
|
||||||
|
if err != nil {
|
||||||
|
return &LoadError{
|
||||||
|
Cause: BadOffsetError,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkOffset(perr *error, filename string, data *FileData, fileSize uint64) {
|
||||||
|
if *perr != nil || data == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if data.Offset+data.Length > fileSize {
|
||||||
|
*perr = &LoadError{
|
||||||
|
Cause: BadOffsetError,
|
||||||
|
Path: filename,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadError reports a problem interpreting the header of a pack file.
|
||||||
|
type LoadError struct {
|
||||||
|
// Cause of the error.
|
||||||
|
Cause ErrorCause
|
||||||
|
|
||||||
|
// Underlying may be set if there is some more information about the
|
||||||
|
// error (e.g. I/O error).
|
||||||
|
Underlying error
|
||||||
|
|
||||||
|
// Magic as read from the file.
|
||||||
|
Magic uint64
|
||||||
|
|
||||||
|
// Version as read from the file.
|
||||||
|
Version uint64
|
||||||
|
|
||||||
|
// Path of an individual file within the pack, if relevant.
|
||||||
|
Path string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrorCause enumerates the possible reasons for failure.
|
||||||
|
type ErrorCause int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// HeaderUnmarshalError means we could not unmarshal the protobuf object
|
||||||
|
// at the head of the file.
|
||||||
|
HeaderUnmarshalError ErrorCause = iota
|
||||||
|
|
||||||
|
// DirectoryUnmarshalError means we could not unmarshal the protobuf
|
||||||
|
// object holding the directory contents.
|
||||||
|
DirectoryUnmarshalError
|
||||||
|
|
||||||
|
// IOError indicates we could not read a protobuf object.
|
||||||
|
IOError
|
||||||
|
|
||||||
|
// MagicMismatch occurs if the recorded magic number does not match
|
||||||
|
// the well-known constant Magic.
|
||||||
|
MagicMismatch
|
||||||
|
|
||||||
|
// VersionTooNew indicates that the file has a version number ahead of
|
||||||
|
// what this package can parse.
|
||||||
|
VersionTooNew
|
||||||
|
|
||||||
|
// VersionTooOld indicates that the file has a version number older than
|
||||||
|
// thisk package can parse.
|
||||||
|
VersionTooOld
|
||||||
|
|
||||||
|
// BadOffsetError means that the header or directory has indicated a
|
||||||
|
// position that lies outside of the file.
|
||||||
|
BadOffsetError
|
||||||
|
|
||||||
|
// InvalidPath is returned for a duplicate or otherwise invalid path.
|
||||||
|
// Underlying is set to a free-form string error describing the path.
|
||||||
|
InvalidPath
|
||||||
|
|
||||||
|
// MissingUncompressed indicates that a file in the pack does not have
|
||||||
|
// an uncompressed version present, which is mandatory.
|
||||||
|
MissingUncompressed
|
||||||
|
)
|
||||||
|
|
||||||
|
// Desc returns a description of the error cause.
|
||||||
|
func (le *LoadError) Desc() string {
|
||||||
|
switch le.Cause {
|
||||||
|
case HeaderUnmarshalError:
|
||||||
|
return "not a .htpack file (header not valid packed.Header)"
|
||||||
|
case DirectoryUnmarshalError:
|
||||||
|
return "file corrupt (directory not valid packed.Directory)"
|
||||||
|
case IOError:
|
||||||
|
return "error reading from file"
|
||||||
|
case MagicMismatch:
|
||||||
|
return "magic number does not match"
|
||||||
|
case VersionTooNew:
|
||||||
|
return "version too new"
|
||||||
|
case VersionTooOld:
|
||||||
|
return "version too old"
|
||||||
|
case BadOffsetError:
|
||||||
|
return "file corrupt/truncated (offset past end of file)"
|
||||||
|
case InvalidPath:
|
||||||
|
return "filename invalid"
|
||||||
|
case MissingUncompressed:
|
||||||
|
return "missing uncompressed version"
|
||||||
|
default:
|
||||||
|
return "unknown error"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error returns a concise description of the error.
|
||||||
|
func (le *LoadError) Error() string {
|
||||||
|
var b strings.Builder
|
||||||
|
|
||||||
|
b.WriteString(le.Desc())
|
||||||
|
|
||||||
|
var underlying, magic, version, path bool
|
||||||
|
switch le.Cause {
|
||||||
|
case HeaderUnmarshalError, IOError:
|
||||||
|
underlying = true
|
||||||
|
case MagicMismatch:
|
||||||
|
magic = true
|
||||||
|
case VersionTooNew, VersionTooOld:
|
||||||
|
version = true
|
||||||
|
case BadOffsetError:
|
||||||
|
path = le.Path != ""
|
||||||
|
case InvalidPath, MissingUncompressed:
|
||||||
|
path = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if underlying {
|
||||||
|
b.WriteString(" [")
|
||||||
|
b.WriteString(le.Underlying.Error())
|
||||||
|
b.WriteString("]")
|
||||||
|
}
|
||||||
|
if magic {
|
||||||
|
fmt.Fprintf(&b, " (found magic 0x%X, expected 0x%X)",
|
||||||
|
le.Magic, uint64(Magic))
|
||||||
|
}
|
||||||
|
if version {
|
||||||
|
fmt.Fprintf(&b, " (found version %d; oldest supported: "+
|
||||||
|
"%d, newest: %d",
|
||||||
|
le.Version, VersionInitial, VersionInitial)
|
||||||
|
}
|
||||||
|
if path {
|
||||||
|
fmt.Fprintf(&b, " (path %q)", le.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
return b.String()
|
||||||
|
}
|
Loading…
Reference in New Issue