Further work-in-progress, basic packing and inspecting

cmd/htpacker/inspector.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/lwithers/htpack/internal/packed"
+)
+
+// Inspect a packfile.
+//  TODO: verify etag; verify integrity of compressed data.
+//  TODO: skip Gzip/Brotli if not present; print ratio.
+func Inspect(filename string) error {
+	f, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	hdr, dir, err := packed.Load(f)
+	if hdr != nil {
+		fmt.Printf("Header: %#v\n", hdr)
+	}
+	if dir != nil {
+		fmt.Printf("%d files:\n", len(dir.Files))
+		for path, info := range dir.Files {
+			fmt.Printf(" • %s\n"+
+				"    · Etag:         %s\n"+
+				"    · Content type: %s\n"+
+				"    · Uncompressed: %s (offset %d)\n"+
+				"    · Gzipped:      %s (offset %d)\n"+
+				"    · Brotli:       %s (offset %d)\n",
+				path, info.Etag, info.ContentType,
+				printSize(info.Uncompressed.Length), info.Uncompressed.Offset,
+				printSize(info.Gzip.Length), info.Gzip.Offset,
+				printSize(info.Brotli.Length), info.Brotli.Offset,
+			)
+		}
+	}
+	return err
+}
+
+func printSize(size uint64) string {
+	switch {
+	case size < 1<<10:
+		return fmt.Sprintf("%d bytes", size)
+	case size < 1<<15:
+		return fmt.Sprintf("%.2f KiB", float64(size)/(1<<10))
+	case size < 1<<20:
+		return fmt.Sprintf("%.1f KiB", float64(size)/(1<<10))
+	case size < 1<<25:
+		return fmt.Sprintf("%.2f MiB", float64(size)/(1<<20))
+	case size < 1<<30:
+		return fmt.Sprintf("%.1f MiB", float64(size)/(1<<20))
+	case size < 1<<35:
+		return fmt.Sprintf("%.2f GiB", float64(size)/(1<<30))
+	default:
+		return fmt.Sprintf("%.1f GiB", float64(size)/(1<<30))
+	}
+}

cmd/htpacker/packer.go

@@ -37,6 +37,7 @@ type packInfo struct {
 	offset, len uint64
 }
 
+// Pack a file.
 func Pack(filesToPack FilesToPack, outputFilename string) error {
 	finalFname, outputFile, err := writefile.New(outputFilename)
 	if err != nil {
@@ -47,8 +48,8 @@ func Pack(filesToPack FilesToPack, outputFilename string) error {
 
 	// write initial header (will rewrite offset/length when known)
 	hdr := &packed.Header{
-		Magic:           123,
-		Version:         1,
+		Magic:           packed.Magic,
+		Version:         packed.VersionInitial,
 		DirectoryOffset: 1,
 		DirectoryLength: 1,
 	}
@@ -117,7 +118,6 @@ func packOne(packer *packWriter, fileToPack FileToPack) (info packed.File, err e
 	}
 	defer unix.Munmap(data)
 
-	// TODO: content-type, etag
 	info.Etag = etag(data)
 	info.ContentType = fileToPack.ContentType
 	if info.ContentType == "" {
@@ -283,6 +283,8 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
 		return 0, pw.err
 	}
 
+	fmt.Fprintf(os.Stderr, "[DEBUG] in size=%d\n", fi.Size())
+
 	var off int64
 	remain := fi.Size()
 	for remain > 0 {
@@ -294,8 +296,9 @@ func (pw *packWriter) CopyFrom(in *os.File) (uint64, error) {
 		}
 
 		amt, err = unix.Sendfile(int(pw.f.Fd()), int(in.Fd()), &off, amt)
+		fmt.Fprintf(os.Stderr, "[DEBUG]   sendfile=%d [off now %d]\n", amt, off)
 		remain -= int64(amt)
-		off += int64(amt)
+		//off += int64(amt)
 		if err != nil {
 			pw.err = err
 			return uint64(off), pw.err

cmd/htpacker/quick_pack.go

@@ -9,13 +9,14 @@ import (
 )
 
 func main() {
-	if err := run(); err != nil {
+	//if err := dopack(); err != nil {
+	if err := Inspect("out.htpack"); err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)
 	}
 }
 
-func run() error {
+func dopack() error {
 	raw, err := ioutil.ReadFile("in.yaml")
 	if err != nil {
 		return err