From cd9c8b600e0626dd51c9e7d682f3b32c51f5d18d Mon Sep 17 00:00:00 2001 From: Laurence Withers Date: Sat, 6 Jun 2026 10:16:40 +0100 Subject: [PATCH] packed: add ability to load from memory, add MapAndLoad The ability to load a packfile from memory means that it becomes easy to use the "embed" package to embed a small packfile. The refactor to allow loading from memory also makes it easy to add a MapAndLoad implementation, which means the file only needs to be opened / mmap()ed once. --- packed/load.go | 135 ++++++++++++++++++++++++-------- packed/load_test.go | 39 +++++++++ packed/testdata/helloworld.pack | Bin 0 -> 16597 bytes packed/testdata/helloworld.txt | 13 +++ 4 files changed, 154 insertions(+), 33 deletions(-) create mode 100644 packed/load_test.go create mode 100644 packed/testdata/helloworld.pack create mode 100644 packed/testdata/helloworld.txt diff --git a/packed/load.go b/packed/load.go index 10609e5..7ea1ade 100644 --- a/packed/load.go +++ b/packed/load.go @@ -2,9 +2,12 @@ package packed import ( fmt "fmt" + "io/fs" "os" "path" "strings" + + "golang.org/x/sys/unix" ) const ( @@ -17,14 +20,14 @@ const ( VersionInitial = 1 ) -// Load a ready-packed file. -func Load(f *os.File) (*Header, *Directory, error) { - hdr, err := loadHeader(f) +// Read a ready-packed file. +func Read(f []byte) (*Header, *Directory, error) { + hdr, err := readHeader(f) if err != nil { return nil, nil, err } - dir, err := loadDirectory(f, hdr) + dir, err := readDirectory(f, hdr) if le, ok := err.(*LoadError); ok { // augment error le.Magic = hdr.Magic @@ -33,20 +36,101 @@ func Load(f *os.File) (*Header, *Directory, error) { return hdr, dir, err // we may have a partial dir } -// loadHeader retrieves and decodes the header from the start of the file. It -// ensures the magic number and the version number match. Errors are returned -// as type LoadError. -func loadHeader(f *os.File) (*Header, error) { - raw := make([]byte, 36) - if _, err := f.ReadAt(raw, 0); err != nil { - return nil, &LoadError{ - Cause: IOError, - Underlying: err, +// Mapped holds an mmap()ed packfile. +type Mapped struct { + Data []byte + Header *Header + Directory *Directory +} + +func (m *Mapped) Close() error { + return unix.Munmap(m.Data) +} + +// MapAndLoad mmaps the named file and reads the pack data. +func MapAndLoad(fname string) (*Mapped, error) { + f, err := os.Open(fname) + if err != nil { + return nil, err + } + defer f.Close() + + st, err := f.Stat() + if err != nil { + return nil, &fs.PathError{ + Path: fname, + Op: "stat", + Err: err, } } + data, err := unix.Mmap(int(f.Fd()), 0, int(st.Size()), unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + return nil, &fs.PathError{ + Path: fname, + Op: "mmap", + Err: err, + } + } + + hdr, dir, err := Read(data) + if err != nil { + unix.Munmap(data) + return nil, err + } + + return &Mapped{ + Data: data, + Header: hdr, + Directory: dir, + }, nil +} + +// Load a ready-packed file. +// +// Deprecated: use [Read] or [MapAndLoad]. +func Load(f *os.File) (*Header, *Directory, error) { + // we need to know the size of the file in order to mmap it + st, err := f.Stat() + if err != nil { + return nil, nil, &fs.PathError{ + Path: f.Name(), + Op: "stat", + Err: err, + } + } + + // compatibility: old code may have worked with non-zero offset + offset, err := f.Seek(0, os.SEEK_CUR) + if err != nil { + return nil, nil, &fs.PathError{ + Path: f.Name(), + Op: "seek", + Err: err, + } + } + + // mmap the file + data, err := unix.Mmap(int(f.Fd()), 0, int(st.Size()), unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + return nil, nil, &fs.PathError{ + Path: f.Name(), + Op: "mmap", + Err: err, + } + } + defer unix.Munmap(data) + + // read as though it's from memory + return Read(data[offset:]) +} + +// readHeader retrieves and decodes the header from the start of the file. It +// ensures the magic number and the version number match. Errors are returned +// as type LoadError. +func readHeader(f []byte) (*Header, error) { hdr := new(Header) - if err := hdr.Unmarshal(raw); err != nil { + if err := hdr.Unmarshal(f[:36]); err != nil { return nil, &LoadError{ Cause: HeaderUnmarshalError, Underlying: err, @@ -77,18 +161,11 @@ func loadHeader(f *os.File) (*Header, error) { return hdr, nil } -// loadDirectory reads the directory from a file. The directory is checked +// readDirectory reads the directory from a file. The directory is checked // for consistency (offsets, filenames) but not integrity (file data is not // read/checksummed). -func loadDirectory(f *os.File, hdr *Header) (*Directory, error) { - fi, err := f.Stat() - if err != nil { - return nil, &LoadError{ - Cause: IOError, - Underlying: err, - } - } - fileSize := uint64(fi.Size()) +func readDirectory(f []byte, hdr *Header) (*Directory, error) { + fileSize := uint64(len(f)) if hdr.DirectoryOffset+hdr.DirectoryLength > fileSize { return nil, &LoadError{ @@ -96,16 +173,8 @@ func loadDirectory(f *os.File, hdr *Header) (*Directory, error) { } } - raw := make([]byte, hdr.DirectoryLength) - if _, err := f.ReadAt(raw, int64(hdr.DirectoryOffset)); err != nil { - return nil, &LoadError{ - Cause: IOError, - Underlying: err, - } - } - dir := new(Directory) - if err := dir.Unmarshal(raw); err != nil { + if err := dir.Unmarshal(f[hdr.DirectoryOffset : hdr.DirectoryOffset+hdr.DirectoryLength]); err != nil { return nil, &LoadError{ Cause: DirectoryUnmarshalError, Underlying: err, diff --git a/packed/load_test.go b/packed/load_test.go new file mode 100644 index 0000000..63be82e --- /dev/null +++ b/packed/load_test.go @@ -0,0 +1,39 @@ +package packed + +import ( + "bytes" + "os" + "testing" +) + +func TestMapAndLoad(t *testing.T) { + rawPack, err := os.ReadFile("testdata/helloworld.pack") + if err != nil { + t.Fatal(err) + } + rawText, err := os.ReadFile("testdata/helloworld.txt") + if err != nil { + t.Fatal(err) + } + + mapped, err := MapAndLoad("testdata/helloworld.pack") + if err != nil { + t.Fatal(err) + } + defer mapped.Close() + + if !bytes.Equal(mapped.Data, rawPack) { + t.Error("mapped data does not match raw file") + } + + file := mapped.Directory.Files["/helloworld.txt"] + if file == nil { + t.Fatal("directory does not contain /helloworld.txt") + } + + start := file.Uncompressed.Offset + end := start + file.Uncompressed.Length + if !bytes.Equal(rawText, mapped.Data[start:end]) { + t.Error("uncompressed file does not match testdata/helloworld.txt") + } +} diff --git a/packed/testdata/helloworld.pack b/packed/testdata/helloworld.pack new file mode 100644 index 0000000000000000000000000000000000000000..8d8541ec8ac97cd566f6031a9d93a0fe96692acb GIT binary patch literal 16597 zcmeI(O-K}B7zc1yDRi;SC8{-|w~fBc)!Eq(N6gSuf|Oc89@3hRcXmdm9hsSJ^&rHG z2uV?=GB1_YrHn!{LMW5kLnW40Y#&fwq5{!AC`$TRbv<_54#IyfGxI+@@ALkc)9^l7 zrDF{hReA6Fb8TsJmou$=?qS`tWP%$6AOHafKmY;|fB*y_009U<00IkJppt5uUO*zc zp-Fc8UPU#DtZI~qddT9{kV$wV(+JVj5H*P|lO`>Z5{oul1;n7drnQj9uxSw*(!)W; zZt27_)L@Vr#8N16iTa@?gPNwQhi2BWl!%h3?ji~e8>(rkVj&j&zq||k^F;R$fB*y_ z009U<00Izz00bbAB7xO!ZRN+F*6xgrTW0HqB_E!_B=@(M|j`sdod6q~2%|*LD{`#}oc|PVG^7^W7K8%bV__?^UZ6rEAKXj0S zouC8+AOHafKmY;|fB*y_009U<00RFKu#ej8IYr9M6u|5ZK%v!aId0qR&Y2_rMYE^< zONprPhDoimuqC@W2X%SeZibir9)=OUoFo);l1KJk@nbjq;zqCqBApigX literal 0 HcmV?d00001 diff --git a/packed/testdata/helloworld.txt b/packed/testdata/helloworld.txt new file mode 100644 index 0000000..3898fd6 --- /dev/null +++ b/packed/testdata/helloworld.txt @@ -0,0 +1,13 @@ +Hello, world + +This file contains a few lines of plain text, really just enough +to trigger the "compression worthwhile" heuristic. + +This file contains a few lines of plain text, really just enough +to trigger the "compression worthwhile" heuristic. + +This file contains a few lines of plain text, really just enough +to trigger the "compression worthwhile" heuristic. + +This file contains a few lines of plain text, really just enough +to trigger the "compression worthwhile" heuristic.