packed: add ability to load from memory, add MapAndLoad

The ability to load a packfile from memory means that it becomes easy to
use the "embed" package to embed a small packfile.

The refactor to allow loading from memory also makes it easy to add a
MapAndLoad implementation, which means the file only needs to be opened
/ mmap()ed once.

packed/load.go

@@ -2,9 +2,12 @@ package packed
 
 import (
 	fmt "fmt"
+	"io/fs"
 	"os"
 	"path"
 	"strings"
+
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -17,14 +20,14 @@ const (
 	VersionInitial = 1
 )
 
-// Load a ready-packed file.
+// Read a ready-packed file.
-func Load(f *os.File) (*Header, *Directory, error) {
+func Read(f []byte) (*Header, *Directory, error) {
-	hdr, err := loadHeader(f)
+	hdr, err := readHeader(f)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	dir, err := loadDirectory(f, hdr)
+	dir, err := readDirectory(f, hdr)
 	if le, ok := err.(*LoadError); ok {
 		// augment error
 		le.Magic = hdr.Magic
@@ -33,20 +36,101 @@ func Load(f *os.File) (*Header, *Directory, error) {
 	return hdr, dir, err // we may have a partial dir
 }
 
-// loadHeader retrieves and decodes the header from the start of the file. It
+// Mapped holds an mmap()ed packfile.
-// ensures the magic number and the version number match. Errors are returned
+type Mapped struct {
-// as type LoadError.
+	Data      []byte
-func loadHeader(f *os.File) (*Header, error) {
+	Header    *Header
-	raw := make([]byte, 36)
+	Directory *Directory
-	if _, err := f.ReadAt(raw, 0); err != nil {
+}
-		return nil, &LoadError{
+
-			Cause:      IOError,
+func (m *Mapped) Close() error {
-			Underlying: err,
+	return unix.Munmap(m.Data)
+}
+
+// MapAndLoad mmaps the named file and reads the pack data.
+func MapAndLoad(fname string) (*Mapped, error) {
+	f, err := os.Open(fname)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	st, err := f.Stat()
+	if err != nil {
+		return nil, &fs.PathError{
+			Path: fname,
+			Op:   "stat",
+			Err:  err,
 		}
 	}
 
+	data, err := unix.Mmap(int(f.Fd()), 0, int(st.Size()), unix.PROT_READ, unix.MAP_SHARED)
+	if err != nil {
+		return nil, &fs.PathError{
+			Path: fname,
+			Op:   "mmap",
+			Err:  err,
+		}
+	}
+
+	hdr, dir, err := Read(data)
+	if err != nil {
+		unix.Munmap(data)
+		return nil, err
+	}
+
+	return &Mapped{
+		Data:      data,
+		Header:    hdr,
+		Directory: dir,
+	}, nil
+}
+
+// Load a ready-packed file.
+//
+// Deprecated: use [Read] or [MapAndLoad].
+func Load(f *os.File) (*Header, *Directory, error) {
+	// we need to know the size of the file in order to mmap it
+	st, err := f.Stat()
+	if err != nil {
+		return nil, nil, &fs.PathError{
+			Path: f.Name(),
+			Op:   "stat",
+			Err:  err,
+		}
+	}
+
+	// compatibility: old code may have worked with non-zero offset
+	offset, err := f.Seek(0, os.SEEK_CUR)
+	if err != nil {
+		return nil, nil, &fs.PathError{
+			Path: f.Name(),
+			Op:   "seek",
+			Err:  err,
+		}
+	}
+
+	// mmap the file
+	data, err := unix.Mmap(int(f.Fd()), 0, int(st.Size()), unix.PROT_READ, unix.MAP_SHARED)
+	if err != nil {
+		return nil, nil, &fs.PathError{
+			Path: f.Name(),
+			Op:   "mmap",
+			Err:  err,
+		}
+	}
+	defer unix.Munmap(data)
+
+	// read as though it's from memory
+	return Read(data[offset:])
+}
+
+// readHeader retrieves and decodes the header from the start of the file. It
+// ensures the magic number and the version number match. Errors are returned
+// as type LoadError.
+func readHeader(f []byte) (*Header, error) {
 	hdr := new(Header)
-	if err := hdr.Unmarshal(raw); err != nil {
+	if err := hdr.Unmarshal(f[:36]); err != nil {
 		return nil, &LoadError{
 			Cause:      HeaderUnmarshalError,
 			Underlying: err,
@@ -77,18 +161,11 @@ func loadHeader(f *os.File) (*Header, error) {
 	return hdr, nil
 }
 
-// loadDirectory reads the directory from a file. The directory is checked
+// readDirectory reads the directory from a file. The directory is checked
 // for consistency (offsets, filenames) but not integrity (file data is not
 // read/checksummed).
-func loadDirectory(f *os.File, hdr *Header) (*Directory, error) {
+func readDirectory(f []byte, hdr *Header) (*Directory, error) {
-	fi, err := f.Stat()
+	fileSize := uint64(len(f))
-	if err != nil {
-		return nil, &LoadError{
-			Cause:      IOError,
-			Underlying: err,
-		}
-	}
-	fileSize := uint64(fi.Size())
 
 	if hdr.DirectoryOffset+hdr.DirectoryLength > fileSize {
 		return nil, &LoadError{
@@ -96,16 +173,8 @@ func loadDirectory(f *os.File, hdr *Header) (*Directory, error) {
 		}
 	}
 
-	raw := make([]byte, hdr.DirectoryLength)
-	if _, err := f.ReadAt(raw, int64(hdr.DirectoryOffset)); err != nil {
-		return nil, &LoadError{
-			Cause:      IOError,
-			Underlying: err,
-		}
-	}
-
 	dir := new(Directory)
-	if err := dir.Unmarshal(raw); err != nil {
+	if err := dir.Unmarshal(f[hdr.DirectoryOffset : hdr.DirectoryOffset+hdr.DirectoryLength]); err != nil {
 		return nil, &LoadError{
 			Cause:      DirectoryUnmarshalError,
 			Underlying: err,

packed/load_test.go

@@ -0,0 +1,39 @@
+package packed
+
+import (
+	"bytes"
+	"os"
+	"testing"
+)
+
+func TestMapAndLoad(t *testing.T) {
+	rawPack, err := os.ReadFile("testdata/helloworld.pack")
+	if err != nil {
+		t.Fatal(err)
+	}
+	rawText, err := os.ReadFile("testdata/helloworld.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mapped, err := MapAndLoad("testdata/helloworld.pack")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer mapped.Close()
+
+	if !bytes.Equal(mapped.Data, rawPack) {
+		t.Error("mapped data does not match raw file")
+	}
+
+	file := mapped.Directory.Files["/helloworld.txt"]
+	if file == nil {
+		t.Fatal("directory does not contain /helloworld.txt")
+	}
+
+	start := file.Uncompressed.Offset
+	end := start + file.Uncompressed.Length
+	if !bytes.Equal(rawText, mapped.Data[start:end]) {
+		t.Error("uncompressed file does not match testdata/helloworld.txt")
+	}
+}

packed/testdata/helloworld.txt

@@ -0,0 +1,13 @@
+Hello, world
+
+This file contains a few lines of plain text, really just enough
+to trigger the "compression worthwhile" heuristic.
+
+This file contains a few lines of plain text, really just enough
+to trigger the "compression worthwhile" heuristic.
+
+This file contains a few lines of plain text, really just enough
+to trigger the "compression worthwhile" heuristic.
+
+This file contains a few lines of plain text, really just enough
+to trigger the "compression worthwhile" heuristic.