diff --git a/README.md b/README.md index f7f576f..b081a09 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # HTTP resource pack server +[![GoDoc](https://img.shields.io/static/v1?label=godoc&message=reference&color=blue)](https://pkg.go.dev/src.lwithers.me.uk/go/htpack) + A common scenario is that you have a set of static resources that you want to serve up quickly via HTTP (for example: stylesheets, WASM). @@ -28,3 +30,42 @@ else will be ignored. The interaction between range handling and compression also seems a little ill-defined; as we have pre-compressed data, however, we can consistently serve the exact same byte data for compressed files. + +## Angular-style single-page application handling + +If you wish to support an angular.js-style single page application, in which +a Javascript application uses the browser's history API to create a set of +virtual paths ("routes"), it is necessary to somehow intercept HTTP 404 errors +being returned from the handler and instead return an HTTP 200 with an HTML +document. + +This can be achieved with a number of methods. + +The simplest method is to tell `packserver` itself which resource to use +instead of returning an HTTP 404. Use the command line argument +`--fallback-404 /index.html` (or whichever named resource). The filename must +match a packed resource, so it will be preceded with a `/`. It must exist in +all packfiles being served. + +If you have an nginx instance reverse proxying in front of `htpack`, then you +can use a couple of extra directives. This is very flexible as it lets you +override the resource for different routes. For example: + + # prevent page loaded at "http://server.example/my-application" from + # requesting resources at "/*" when it should request them at + # "/my-application/*" instead + location = /my-application { + return 308 /my-application/; + } + + location /my-application/ { + proxy_to http://htpack-addr:8080/; + proxy_intercept_errors on; + error_page 404 =200 /my-application/; + } + +If you are using the handler as a library, then you may call +`handler.SetNotFound(filename)` to select a resource to return (with HTTP 200) +if a request is made for a resource that is not found. The filename must match +a packed resource, so it will be preceded with a `/` (for example it may be +`"/index.html"`). diff --git a/cmd/htpacker/content_type_glob.go b/cmd/htpacker/content_type_glob.go new file mode 100644 index 0000000..e461908 --- /dev/null +++ b/cmd/htpacker/content_type_glob.go @@ -0,0 +1,105 @@ +package main + +import ( + "fmt" + "path/filepath" + "strings" + + "src.lwithers.me.uk/go/htpack/cmd/htpacker/packer" +) + +type ctGlobEntry struct { + pattern, contentType string + pathComponents int +} + +type ctGlobList []ctGlobEntry + +func parseGlobs(flags []string) (ctGlobList, error) { + var ctGlobs ctGlobList + for _, flag := range flags { + // split pattern:content-type + pos := strings.LastIndexByte(flag, ':') + if pos == -1 { + return nil, &parseGlobError{ + Value: flag, + Err: "must be pattern:content-type", + } + } + pattern, ct := flag[:pos], flag[pos+1:] + + // patterns starting with "/" must match the entire directory + // prefix; otherwise, an arbitrary number of path components are + // allowed prior to the prefix + var pathComponents int + if strings.HasPrefix(pattern, "/") { + pathComponents = -1 + pattern = strings.TrimPrefix(pattern, "/") + } else { + pathComponents = 1 + strings.Count(pattern, "/") + } + + // test that the pattern's syntax is valid + if _, err := filepath.Match(pattern, "test"); err != nil { + return nil, &parseGlobError{ + Value: flag, + Err: err.Error(), + } + } + + ctGlobs = append(ctGlobs, ctGlobEntry{ + pattern: pattern, + contentType: ct, + pathComponents: pathComponents, + }) + } + + return ctGlobs, nil +} + +// ApplyContentTypes will scan the list of files to pack, matching by filename, +// and on match will apply the given content type. +func (ctGlobs ctGlobList) ApplyContentTypes(ftp packer.FilesToPack) { + for name := range ftp { + for _, entry := range ctGlobs { + testName := trimPathComponents(name, entry.pathComponents) + matched, _ := filepath.Match(entry.pattern, testName) + if matched { + f := ftp[name] + f.ContentType = entry.contentType + ftp[name] = f + break + } + } + } +} + +func trimPathComponents(name string, components int) string { + name = strings.TrimPrefix(name, "/") // FilesToPack keys = absolute path + + // if we are matching the full prefix, don't otherwise manipulate the + // name + if components < 0 { + return name + } + + // otherwise, trim the number of components remaining in the path so + // that we are only matching the trailing path components from the + // FilesToPack key + parts := 1 + strings.Count(name, "/") + for ; parts > components; parts-- { + pos := strings.IndexByte(name, '/') + name = name[pos+1:] + } + return name +} + +// parseGlobError is returned from parseGlobs on error. +type parseGlobError struct { + Value string + Err string +} + +func (pge *parseGlobError) Error() string { + return fmt.Sprintf("--content-type entry %q: %s", pge.Value, pge.Err) +} diff --git a/cmd/htpacker/content_type_glob_test.go b/cmd/htpacker/content_type_glob_test.go new file mode 100644 index 0000000..7cc2822 --- /dev/null +++ b/cmd/htpacker/content_type_glob_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "testing" + + "src.lwithers.me.uk/go/htpack/cmd/htpacker/packer" +) + +func TestParseGlobs(t *testing.T) { + ctGlobs, err := parseGlobs([]string{ + "*.foo:text/html", + "*.bar:text/plain", + "baz/qux/*.js:application/javascript", + "/abs/file:image/png", + }) + if err != nil { + t.Fatal(err) + } + + check := func(pos int, pattern, contentType string, pathComponents int) { + if pos >= len(ctGlobs) { + t.Errorf("entry %d not present", pos) + return + } + + if pattern != ctGlobs[pos].pattern { + t.Errorf("entry %d: expected pattern %q but got %q", + pos, pattern, ctGlobs[pos].pattern) + } + + if contentType != ctGlobs[pos].contentType { + t.Errorf("entry %d: expected content type %q but got %q", + pos, contentType, ctGlobs[pos].contentType) + } + + if pathComponents != ctGlobs[pos].pathComponents { + t.Errorf("entry %d: expected num. path components %d but got %d", + pos, pathComponents, ctGlobs[pos].pathComponents) + } + } + + check(0, "*.foo", "text/html", 1) + check(1, "*.bar", "text/plain", 1) + check(2, "baz/qux/*.js", "application/javascript", 3) + check(3, "abs/file", "image/png", -1) +} + +func TestParseGlobsErrSep(t *testing.T) { + const badValue = "hello/dave.js" // missing ":" separator + _, err := parseGlobs([]string{badValue}) + switch err := err.(type) { + case *parseGlobError: + if err.Value != badValue { + t.Errorf("expected value %q but got %q", badValue, err.Value) + } + case nil: + t.Fatal("expected error") + default: + t.Errorf("unexpected error type %T (value %v)", err, err) + } +} + +func TestParseGlobsErrPattern(t *testing.T) { + const badValue = "[-z]:foo/bar" // malformed character class + _, err := parseGlobs([]string{badValue}) + switch err := err.(type) { + case *parseGlobError: + if err.Value != badValue { + t.Errorf("expected value %q but got %q", badValue, err.Value) + } + case nil: + t.Fatal("expected error") + default: + t.Errorf("unexpected error type %T (value %v)", err, err) + } +} + +func TestApplyContentTypes(t *testing.T) { + // XXX: we program our _expectation_ of content-type into the Filename field + ftp := packer.FilesToPack{ + "foo.txt": packer.FileToPack{Filename: "text/plain"}, + "baz/foo.txt": packer.FileToPack{Filename: "text/plain"}, + + "baz/qux.png": packer.FileToPack{Filename: "image/png"}, + "foo/qux.png": packer.FileToPack{}, + "foo/baz/qux.png": packer.FileToPack{Filename: "image/png"}, + + "bar.jpeg": packer.FileToPack{}, + "foo/baz/bar.jpeg": packer.FileToPack{}, + "baz/bar.jpeg": packer.FileToPack{Filename: "image/jpeg"}, + } + + ctGlobs, err := parseGlobs([]string{ + "*.txt:text/plain", // should match anywhere + "baz/qux.png:image/png", // won't match /foo/qux.png + "/baz/bar.jpeg:image/jpeg", // exact prefix match + }) + if err != nil { + t.Fatal(err) + } + + ctGlobs.ApplyContentTypes(ftp) + for k, v := range ftp { + if v.Filename != v.ContentType { + t.Errorf("filename %q: expected content type %q but got %q", + k, v.Filename, v.ContentType) + } + } +} diff --git a/cmd/htpacker/go.mod b/cmd/htpacker/go.mod index 6c5be0a..889caa3 100644 --- a/cmd/htpacker/go.mod +++ b/cmd/htpacker/go.mod @@ -1,17 +1,24 @@ module src.lwithers.me.uk/go/htpack/cmd/htpacker -go 1.12 +go 1.22 require ( - github.com/andybalholm/brotli v1.0.0 + github.com/andybalholm/brotli v1.1.0 github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2 - github.com/kr/pretty v0.1.0 // indirect - github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 - github.com/spf13/cobra v0.0.5 - github.com/vbauerster/mpb/v4 v4.11.2 - golang.org/x/sys v0.0.0-20200116001909-b77594299b42 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - gopkg.in/yaml.v2 v2.2.2 - src.lwithers.me.uk/go/htpack v1.1.5 + github.com/gosuri/uiprogress v0.0.1 + github.com/spf13/cobra v1.8.1 + golang.org/x/sys v0.22.0 + gopkg.in/yaml.v2 v2.4.0 + src.lwithers.me.uk/go/htpack v1.3.3 src.lwithers.me.uk/go/writefile v1.0.1 ) + +require ( + github.com/gogo/protobuf v1.3.2 // indirect + github.com/gosuri/uilive v0.0.4 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/kr/pretty v0.1.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/spf13/pflag v1.0.5 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect +) diff --git a/cmd/htpacker/go.sum b/cmd/htpacker/go.sum index d605785..d0f5e7b 100644 --- a/cmd/htpacker/go.sum +++ b/cmd/htpacker/go.sum @@ -1,75 +1,68 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= -github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= -github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2 h1:VA6jElpcJ+wkwEBufbnVkSBCA2TEnxdRppjRT5Kvh0A= github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2/go.mod h1:Yi95+RbwKz7uGndSuUhoq7LJKh8qH8DT9fnL4ewU30k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= +github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= +github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= +github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= -github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/vbauerster/mpb/v4 v4.11.2 h1:ynkUoKzi65DZ1UsQPx7sgi/KN6G9f7br+Us2nKm35AM= -github.com/vbauerster/mpb/v4 v4.11.2/go.mod h1:jIuIRCltGJUnm6DCyPVkwjlLUk4nHTH+m4eD14CdFF0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= -golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 h1:gZpLHxUX5BdYLA08Lj4YCJNN/jk7KtquiArPoeX0WvA= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -src.lwithers.me.uk/go/htpack v1.1.5 h1:2JzgqLZ1ROYc53+96NezfJ3S9fkwHZNd6QgJhMXnlSE= -src.lwithers.me.uk/go/htpack v1.1.5/go.mod h1:JWofpm01RJbCTIyKfIPftUsxk6KlFkrYwyHgCVdKY+s= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +src.lwithers.me.uk/go/htpack v1.3.3 h1:Xvl6qR9HfSblmCgPyu+ACQ9o3aLQSIy3l8CrMbzj/jc= +src.lwithers.me.uk/go/htpack v1.3.3/go.mod h1:qKgCBgZ6iiiuYOxZkYOPVpXLBzp6gXEd4A0ksxgR6Nk= src.lwithers.me.uk/go/writefile v1.0.1 h1:bwBGtvyZfCxFIM14e1aYgJWlZuowKkwJx53OJlUPd0s= src.lwithers.me.uk/go/writefile v1.0.1/go.mod h1:NahlmRCtB7kg4ai+zHZgxXdUs+MR8VqWG8mql35TsxA= diff --git a/cmd/htpacker/pack.go b/cmd/htpacker/pack.go index f584616..e829950 100644 --- a/cmd/htpacker/pack.go +++ b/cmd/htpacker/pack.go @@ -16,6 +16,20 @@ import ( var packCmd = &cobra.Command{ Use: "pack", Short: "creates a packfile from a YAML spec or set of files/dirs", + Long: `When given a YAML spec file (a template for which can be generated +with the "yaml" command), files will be packed exactly as per the spec. The +--content-type flag cannot be used and no extra files can be specified. + +When given a list of files and directories to pack, the content type for each +file will be automatically detected. It is possible to override the content +type by specifying one or more --content-type flags. These take an argument in +the form "pattern:content/type". The pattern is matched using common glob +(* = wildcard), very similar to .gitignore. If the pattern contains any +directory names, these must match the final components of the file to pack's +path. If the pattern starts with a "/", then the full path must be matched +exactly. +`, + RunE: func(c *cobra.Command, args []string) error { // convert "out" to an absolute path, so that it will still // work after chdir @@ -51,6 +65,16 @@ var packCmd = &cobra.Command{ } } + // parse content-type globs + ctGlobList, err := c.Flags().GetStringArray("content-type") + if err != nil { + return err + } + ctGlobs, err := parseGlobs(ctGlobList) + if err != nil { + return err + } + // if "spec" is not present, then we expect a list of input // files, and we'll build a spec from them if spec == "" { @@ -58,12 +82,16 @@ var packCmd = &cobra.Command{ return errors.New("need --yaml, " + "or one or more filenames") } - err = PackFiles(c, args, out) + err = PackFiles2(c, args, ctGlobs, out) } else { if len(args) != 0 { return errors.New("cannot specify files " + "when using --yaml") } + if ctGlobs != nil { + return errors.New("cannot specify --content-type " + + "when using --yaml") + } err = PackSpec(c, spec, out) } if err != nil { @@ -83,13 +111,20 @@ func init() { "YAML specification file (if not present, just pack files)") packCmd.Flags().StringP("chdir", "C", "", "Change to directory before searching for input files") + packCmd.Flags().StringArrayP("content-type", "", nil, + "Override content type for pattern, e.g. \"*.foo=bar/baz\" (like .gitignore)") } func PackFiles(c *cobra.Command, args []string, out string) error { + return PackFiles2(c, args, nil, out) +} + +func PackFiles2(c *cobra.Command, args []string, ctGlobs ctGlobList, out string) error { ftp, err := filesFromList(args) if err != nil { return err } + ctGlobs.ApplyContentTypes(ftp) return doPack(ftp, out) } @@ -109,7 +144,7 @@ func PackSpec(c *cobra.Command, spec, out string) error { } func doPack(ftp packer.FilesToPack, out string) error { - prog := mpbProgress(ftp) + prog := newUiProgress(ftp) err := packer.Pack2(ftp, out, prog) prog.Complete() diff --git a/cmd/htpacker/pack_progress.go b/cmd/htpacker/pack_progress.go deleted file mode 100644 index 9f7ce88..0000000 --- a/cmd/htpacker/pack_progress.go +++ /dev/null @@ -1,144 +0,0 @@ -package main - -import ( - "github.com/logrusorgru/aurora" - "github.com/vbauerster/mpb/v4" - "github.com/vbauerster/mpb/v4/decor" - "src.lwithers.me.uk/go/htpack/cmd/htpacker/packer" -) - -const mpbBarStyle = "[██░]" - -// mpbProgress returns a new progress object that keeps the user informed via -// the visual mpb library. -func mpbProgress(ftp packer.FilesToPack) *mpbProg { - mp := new(mpbProg) - mp.un.max = len(ftp) - - for _, f := range ftp { - if !f.DisableCompression && !f.DisableGzip { - mp.gzip.max++ - } - if !f.DisableCompression && !f.DisableBrotli { - mp.brotli.max++ - } - } - - mp.p = mpb.New() - mp.un.bar = mp.p.AddBar(int64(mp.un.max), - mpb.PrependDecorators(barName("uncompressed")), - mpb.BarStyle(mpbBarStyle), - mpb.AppendDecorators(&mp.un)) - if mp.gzip.max > 0 { - mp.gzip.bar = mp.p.AddBar(int64(mp.gzip.max), - mpb.PrependDecorators(barName("gzip")), - mpb.BarStyle(mpbBarStyle), - mpb.AppendDecorators(&mp.gzip)) - } - if mp.brotli.max > 0 { - mp.brotli.bar = mp.p.AddBar(int64(mp.brotli.max), - mpb.PrependDecorators(barName("brotli")), - mpb.BarStyle(mpbBarStyle), - mpb.AppendDecorators(&mp.brotli)) - } - - return mp -} - -func barName(n string) decor.Decorator { - return decor.Name(aurora.Magenta(n).String(), decor.WCSyncWidth) -} - -// mpbProg is the mpb progress tracker. It has one bar for each type of -// compression, and its methods simply dispatch onto the type-specific -// bars. -type mpbProg struct { - un, gzip, brotli mpbProg1 - p *mpb.Progress -} - -func (mp *mpbProg) Count(_ int) { -} - -func (mp *mpbProg) Begin(filename, compression string) { - switch compression { - case "uncompressed": - mp.un.Begin(filename) - case "gzip": - mp.gzip.Begin(filename) - case "brotli": - mp.brotli.Begin(filename) - default: - return - } -} - -func (mp *mpbProg) End(filename, compression string) { - switch compression { - case "uncompressed": - mp.un.End(filename) - case "gzip": - mp.gzip.End(filename) - case "brotli": - mp.brotli.End(filename) - default: - return - } -} - -func (mp *mpbProg) Complete() { - mp.un.Complete() - mp.gzip.Complete() - mp.brotli.Complete() - mp.p.Wait() -} - -// mpbProg1 is a type-specific progress bar. In addition to holding state and -// methods for updating the bar, it also implements decor.Decor. -type mpbProg1 struct { - max int // number of items we expect - done int // number of items completed - cur []string // list of currently-packing filenames - bar *mpb.Bar - - // embedding this type lets us implement decor.Decor - decor.WC -} - -func (mp1 *mpbProg1) Decor(stat *decor.Statistics) string { - if stat.Completed { - return "" - } - switch len(mp1.cur) { - case 0: - return aurora.Gray(8, "(idle)").String() - case 1: - return aurora.Blue(mp1.cur[0]).String() - default: - return aurora.Sprintf(aurora.Green("%s + %d more"), aurora.Blue(mp1.cur[0]), len(mp1.cur)-1) - } -} - -func (mp1 *mpbProg1) Begin(filename string) { - mp1.cur = append(mp1.cur, filename) -} - -func (mp1 *mpbProg1) End(filename string) { - for i, v := range mp1.cur { - if v == filename { - mp1.cur[i] = mp1.cur[len(mp1.cur)-1] - mp1.cur = mp1.cur[:len(mp1.cur)-1] - break - } - } - mp1.done++ - if mp1.bar != nil { - mp1.bar.SetCurrent(int64(mp1.done)) - } -} - -func (mp1 *mpbProg1) Complete() { - if mp1.bar != nil { - mp1.bar.SetTotal(int64(mp1.max), true) - } -} diff --git a/cmd/htpacker/packer/packer.go b/cmd/htpacker/packer/packer.go index 00cc31e..957a217 100644 --- a/cmd/htpacker/packer/packer.go +++ b/cmd/htpacker/packer/packer.go @@ -64,6 +64,11 @@ func (ignoreProgress) Begin(_, _ string) {} func (ignoreProgress) End(_, _ string) {} const ( + // minCompressionFileSize is the minimum filesize we need before + // considering compression. Note this must be at least 2, to avoid + // known bugs in go-zopfli. + minCompressionFileSize = 128 + // minCompressionSaving means we'll only use the compressed version of // the file if it's at least this many bytes smaller than the original. // Chosen somewhat arbitrarily; we have to add an HTTP header, and the @@ -325,11 +330,14 @@ func (p *packer) packFile(path string, fileToPack FileToPack) { return } - data, err := unix.Mmap(int(f.Fd()), 0, int(fi.Size()), - unix.PROT_READ, unix.MAP_SHARED) - if err != nil { - p.Abort(fmt.Errorf("mmap %s: %v", fileToPack.Filename, err)) - return + var data []byte + if fi.Size() > 0 { + data, err = unix.Mmap(int(f.Fd()), 0, int(fi.Size()), + unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + p.Abort(fmt.Errorf("mmap %s: %v", fileToPack.Filename, err)) + return + } } // prepare initial directory entry @@ -356,7 +364,7 @@ func (p *packer) packFile(path string, fileToPack FileToPack) { defer p.progress.End(fileToPack.Filename, "gzip") if err := p.Gzip(data, info); err != nil { return fmt.Errorf("gzip compression of %s "+ - "failed: %v", fileToPack.Filename, err) + "failed: %w", fileToPack.Filename, err) } return nil }) @@ -367,7 +375,7 @@ func (p *packer) packFile(path string, fileToPack FileToPack) { defer p.progress.End(fileToPack.Filename, "brotli") if err := p.Brotli(data, info); err != nil { return fmt.Errorf("brotli compression of %s "+ - "failed: %v", fileToPack.Filename, err) + "failed: %w", fileToPack.Filename, err) } return nil }) @@ -430,6 +438,10 @@ func (p *packer) Uncompressed(srcPath string, dir *packed.File) error { // Gzip will gzip input data to a temporary file, and then append that to the // output file. func (p *packer) Gzip(data []byte, dir *packed.File) error { + if len(data) < minCompressionFileSize { + return nil + } + // write via temporary file tmpfile, err := ioutil.TempFile("", "") if err != nil { @@ -471,6 +483,9 @@ func (p *packer) Gzip(data []byte, dir *packed.File) error { // Brotli will compress input data to a temporary file, and then append that to // the output file. func (p *packer) Brotli(data []byte, dir *packed.File) error { + if len(data) < minCompressionFileSize { + return nil + } // write via temporary file tmpfile, err := ioutil.TempFile("", "") if err != nil { diff --git a/cmd/htpacker/progress.go b/cmd/htpacker/progress.go new file mode 100644 index 0000000..488d182 --- /dev/null +++ b/cmd/htpacker/progress.go @@ -0,0 +1,131 @@ +package main + +import ( + "bytes" + "slices" + "sync" + + "github.com/gosuri/uiprogress" + "src.lwithers.me.uk/go/htpack/cmd/htpacker/packer" +) + +type uiProgress struct { + p *uiprogress.Progress + uncompressed, gzip, brotli *uiProgressBar +} + +func newUiProgress(ftp packer.FilesToPack) *uiProgress { + up := &uiProgress{ + p: uiprogress.New(), + } + + up.uncompressed = newUiProgressBar(up.p, len(ftp), "uncompressed") + var nGzip, nBrotli int + for _, f := range ftp { + if !f.DisableCompression && !f.DisableGzip { + nGzip++ + } + if !f.DisableCompression && !f.DisableBrotli { + nBrotli++ + } + } + if nGzip > 0 { + up.gzip = newUiProgressBar(up.p, nGzip, "gzip") + } + if nBrotli > 0 { + up.brotli = newUiProgressBar(up.p, nGzip, "brotli") + } + + up.p.Start() + return up +} + +func (up *uiProgress) Count(_ int) { +} + +func (up *uiProgress) Begin(filename, compression string) { + up.bar(compression).begin(filename) +} + +func (up *uiProgress) End(filename, compression string) { + up.bar(compression).end(filename) +} + +func (up *uiProgress) bar(compression string) *uiProgressBar { + switch compression { + case "uncompressed": + return up.uncompressed + case "gzip": + return up.gzip + case "brotli": + return up.brotli + } + return nil +} + +func (up *uiProgress) Complete() { + up.p.Stop() +} + +type uiProgressBar struct { + bar *uiprogress.Bar + lock sync.Mutex + inflight []string +} + +func newUiProgressBar(p *uiprogress.Progress, total int, compression string) *uiProgressBar { + bar := &uiProgressBar{ + bar: p.AddBar(total).AppendCompleted(), + } + + var buf bytes.Buffer + bar.bar.PrependFunc(func(*uiprogress.Bar) string { + bar.lock.Lock() + defer bar.lock.Unlock() + buf.Reset() + buf.WriteString(compression) + if len(bar.inflight) > 0 { + buf.WriteString(" (") + for i, f := range bar.inflight { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(f) + } + buf.WriteRune(')') + } + if buf.Len() < 40 { + buf.WriteString(" ") + buf.Truncate(40) + } else if buf.Len() > 40 { + buf.Truncate(39) + buf.WriteString("…") + } + return buf.String() + }) + + return bar +} + +func (bar *uiProgressBar) begin(filename string) { + if bar == nil { + return + } + bar.lock.Lock() + defer bar.lock.Unlock() + + bar.inflight = append(bar.inflight, filename) +} + +func (bar *uiProgressBar) end(filename string) { + if bar == nil { + return + } + bar.lock.Lock() + defer bar.lock.Unlock() + + bar.bar.Incr() + if idx := slices.Index(bar.inflight, filename); idx != -1 { + bar.inflight = slices.Delete(bar.inflight, idx, idx+1) + } +} diff --git a/cmd/htpacker/yaml.go b/cmd/htpacker/yaml.go index eb8eaba..c20cb02 100644 --- a/cmd/htpacker/yaml.go +++ b/cmd/htpacker/yaml.go @@ -3,6 +3,7 @@ package main import ( "errors" "fmt" + "io" "io/ioutil" "net/http" "os" @@ -125,13 +126,22 @@ func filesFromListR(prefix, arg string, ftp packer.FilesToPack) error { case fi.Mode().IsRegular(): // sniff content type + var ctype string buf := make([]byte, 512) n, err := f.Read(buf) - if err != nil { - return err + switch err { + case nil: + buf = buf[:n] + ctype = http.DetectContentType(buf) + + case io.EOF: + // Empty file; this is typically due to things like + // npm webpack producing empty .css files. + ctype = "text/plain; charset=UTF-8" + + default: + return fmt.Errorf("failed to read %s: %v", arg, err) } - buf = buf[:n] - ctype := http.DetectContentType(buf) // augmented rules for JS / CSS / etc. switch { @@ -140,7 +150,7 @@ func filesFromListR(prefix, arg string, ftp packer.FilesToPack) error { case ".css": ctype = "text/css" case ".js": - ctype = "application/javascript" + ctype = "text/javascript" case ".json": ctype = "application/json" case ".svg": diff --git a/cmd/packserver/go.mod b/cmd/packserver/go.mod index 06f2135..958080f 100644 --- a/cmd/packserver/go.mod +++ b/cmd/packserver/go.mod @@ -1,8 +1,15 @@ module src.lwithers.me.uk/go/htpack/cmd/packserver -go 1.13 +go 1.26.0 require ( - github.com/spf13/cobra v0.0.5 - src.lwithers.me.uk/go/htpack v1.1.5 + github.com/spf13/cobra v1.10.2 + src.lwithers.me.uk/go/htpack v1.5.0 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + golang.org/x/sys v0.47.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/cmd/packserver/go.sum b/cmd/packserver/go.sum index 9c25300..180bbaa 100644 --- a/cmd/packserver/go.sum +++ b/cmd/packserver/go.sum @@ -1,42 +1,19 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 h1:gZpLHxUX5BdYLA08Lj4YCJNN/jk7KtquiArPoeX0WvA= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -src.lwithers.me.uk/go/htpack v1.1.5 h1:2JzgqLZ1ROYc53+96NezfJ3S9fkwHZNd6QgJhMXnlSE= -src.lwithers.me.uk/go/htpack v1.1.5/go.mod h1:JWofpm01RJbCTIyKfIPftUsxk6KlFkrYwyHgCVdKY+s= +src.lwithers.me.uk/go/htpack v1.5.0 h1:2UYM4AGIPc/XgktmO6AxUyqFxzmYkqg/jINcDCsZsCk= +src.lwithers.me.uk/go/htpack v1.5.0/go.mod h1:5dQlPdwV+ZeBAaFBdwiRI6/w5dpSgPsiotO4ZTYau88= diff --git a/cmd/packserver/main.go b/cmd/packserver/main.go index c896e42..a374a66 100644 --- a/cmd/packserver/main.go +++ b/cmd/packserver/main.go @@ -5,12 +5,18 @@ package main import ( "bufio" + "context" "errors" "fmt" "net/http" "os" + "os/signal" "path/filepath" + "sort" "strings" + "sync/atomic" + "syscall" + "time" "github.com/spf13/cobra" "src.lwithers.me.uk/go/htpack" @@ -49,6 +55,16 @@ func main() { "Name of index file (index.html or similar)") rootCmd.Flags().Duration("expiry", 0, "Tell client how long it can cache data for; 0 means no caching") + rootCmd.Flags().Duration("expiry404", 0, + "Tell client how long it can cache 404 responses for; 0 means default (inherit from --expiry, max. 60s); -1 means no caching") + rootCmd.Flags().Bool("immutable", false, + "Tell client that cached assets are immutable (will not be revalidated on page reload)") + rootCmd.Flags().String("fallback-404", "", + "Name of file to return if response would be 404 (spa.html or similar)") + rootCmd.Flags().String("frames", "sameorigin", + "Override X-Frame-Options header (can be sameorigin, deny, allow)") + rootCmd.Flags().Duration("graceful-shutdown-delay", 3*time.Second, + "Number of seconds to wait after receiving SIGTERM before initiating graceful shutdown") if err := rootCmd.Execute(); err != nil { fmt.Fprintln(os.Stderr, err) @@ -80,6 +96,23 @@ func run(c *cobra.Command, args []string) error { certFile = keyFile } + // parse frames header + framesHeader := "SAMEORIGIN" + frames, err := c.Flags().GetString("frames") + if err != nil { + return err + } + switch frames { + case "sameorigin": + framesHeader = "SAMEORIGIN" + case "allow": + framesHeader = "" + case "deny": + framesHeader = "DENY" + default: + return errors.New("--frames must be one of sameorigin, deny, allow") + } + // parse extra headers extraHeaders := make(http.Header) hdrs, err := c.Flags().GetStringSlice("header") @@ -106,15 +139,35 @@ func run(c *cobra.Command, args []string) error { // parse expiry time // NB: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control + cacheControl200, cacheControl404 := "no-cache", "no-cache" + immutable, err := c.Flags().GetBool("immutable") + if err != nil { + return err + } + var immutableDirective string + if immutable { + immutableDirective = ", immutable" + } + expiry, err := c.Flags().GetDuration("expiry") if err != nil { return err } - if expiry <= 0 { - extraHeaders.Set("Cache-Control", "no-cache") - } else { - extraHeaders.Set("Cache-Control", - fmt.Sprintf("public, max-age=%d", expiry/1e9)) + if expiry > 0 { + cacheControl200 = fmt.Sprintf("public, max-age=%d%s", expiry/1e9, immutableDirective) + } + + expiry404, err := c.Flags().GetDuration("expiry404") + if err != nil { + return err + } + switch { + case expiry404 == 0 && expiry == 0: + case expiry404 == 0 && expiry > 0: + expiry404 = min(60*time.Second, expiry) + fallthrough + case expiry404 > 0: + cacheControl404 = fmt.Sprintf("public, max-age=%d", expiry404/1e9) } // optional index file @@ -124,6 +177,21 @@ func run(c *cobra.Command, args []string) error { return err } + // optional 404 fallback file + fallback404File, err := c.Flags().GetString("fallback-404") + if err != nil { + return err + } + + // graceful shutdown delay must be > 0 + gracefulShutdownDelay, err := c.Flags().GetDuration("graceful-shutdown-delay") + if err != nil { + return err + } + if gracefulShutdownDelay <= 0 { + return errors.New("graceful shutdown delay must be > 0s") + } + // verify .htpack specifications if len(args) == 0 { return errors.New("must specify one or more .htpack files") @@ -149,35 +217,75 @@ func run(c *cobra.Command, args []string) error { } // load packfiles, registering handlers as we go + router := &routerHandler{} for prefix, packfile := range packPaths { packHandler, err := htpack.New(packfile) if err != nil { return err } + packHandler.SetCacheControl(cacheControl200, cacheControl404) if indexFile != "" { packHandler.SetIndex(indexFile) } + if err = packHandler.SetNotFound(fallback404File); err != nil { + return fmt.Errorf("%s: fallback-404 resource %q "+ + "not found in packfile", prefix, fallback404File) + } + packHandler.SetHeader("X-Frame-Options", framesHeader) - handler := &addHeaders{ + var handler http.Handler = &addHeaders{ extraHeaders: extraHeaders, handler: packHandler, } if prefix != "/" { - http.Handle(prefix+"/", - http.StripPrefix(prefix, handler)) - } else { - http.Handle("/", handler) + handler = http.StripPrefix(prefix, handler) } + router.AddRoute(prefix, handler) } + // HTTP server object setup + sv := &http.Server{ + Addr: bindAddr, + Handler: router, + } + + // register SIGINT, SIGTERM handler + sigch := make(chan os.Signal, 1) + signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM) + var ( + // if we are shut down by a signal, then http.ListenAndServe() + // returns straight away, but we actually need to wait for + // Shutdown() to complete prior to returning / exiting + isSignalled atomic.Bool + signalDone = make(chan struct{}) + ) + go func() { + <-sigch + time.Sleep(gracefulShutdownDelay) + isSignalled.Store(true) + shutctx, shutcancel := context.WithTimeout(context.Background(), gracefulShutdownDelay) + sv.Shutdown(shutctx) + shutcancel() + close(signalDone) + }() + // main server loop if keyFile == "" { - err = http.ListenAndServe(bindAddr, nil) + err = sv.ListenAndServe() } else { - err = http.ListenAndServeTLS(bindAddr, certFile, keyFile, nil) + err = sv.ListenAndServeTLS(certFile, keyFile) } - if err != nil { + + // if we were shut down by a signal, wait for Shutdown() to return + if isSignalled.Load() { + <-signalDone + } + + switch err { + case nil, http.ErrServerClosed: + // OK + default: fmt.Fprintln(os.Stderr, err) os.Exit(1) } @@ -228,3 +336,53 @@ func (ah *addHeaders) ServeHTTP(w http.ResponseWriter, r *http.Request) { } ah.handler.ServeHTTP(w, r) } + +// routeEntry is used within routerHandler to map a specific prefix to a +// specific handler. +type routeEntry struct { + // prefix is a path prefix with trailing "/" such as "/foo/". + prefix string + + // handler for the request if prefix matches. + handler http.Handler +} + +// routerHandler holds a list of routes sorted by longest-prefix-first. +type routerHandler struct { + // entries are the list of prefixes, with longest prefix strings first. + // The sorting ensures we can iterate through from the start and match + // "/dir/subdir/" in preference to just "/dir/". + entries []routeEntry +} + +// AddRoute adds a new entry into the handler. It is not concurrency safe; the +// handler should not be in use. +func (rh *routerHandler) AddRoute(prefix string, handler http.Handler) { + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + rh.entries = append(rh.entries, routeEntry{ + prefix: prefix, + handler: handler, + }) + sort.Slice(rh.entries, func(i, j int) bool { + l1, l2 := len(rh.entries[i].prefix), len(rh.entries[j].prefix) + if l1 > l2 { + return true + } + if l1 == l2 { + return rh.entries[i].prefix < rh.entries[j].prefix + } + return false + }) +} + +func (rh *routerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + for _, entry := range rh.entries { + if strings.HasPrefix(r.URL.Path, entry.prefix) { + entry.handler.ServeHTTP(w, r) + return + } + } + http.NotFound(w, r) +} diff --git a/go.mod b/go.mod index 5079ce9..075ad33 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module src.lwithers.me.uk/go/htpack -require ( - github.com/gogo/protobuf v1.2.1 - golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 -) +go 1.26.0 -go 1.13 +require ( + golang.org/x/sys v0.47.0 + google.golang.org/protobuf v1.36.12 +) diff --git a/go.sum b/go.sum index faee370..3bd003e 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,6 @@ -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 h1:gZpLHxUX5BdYLA08Lj4YCJNN/jk7KtquiArPoeX0WvA= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/handler.go b/handler.go index a5782a3..9f13f42 100644 --- a/handler.go +++ b/handler.go @@ -2,17 +2,13 @@ package htpack import ( "fmt" - "net" "net/http" - "os" "path" "path/filepath" "strconv" "strings" - "syscall" "time" - "golang.org/x/sys/unix" "src.lwithers.me.uk/go/htpack/packed" ) @@ -21,43 +17,24 @@ const ( encodingBrotli = "br" ) -// TODO: logging - // New returns a new handler. Standard security headers are set. func New(packfile string) (*Handler, error) { - f, err := os.Open(packfile) + mapped, err := packed.MapAndLoad(packfile) if err != nil { return nil, err } - fi, err := f.Stat() - if err != nil { - return nil, err - } - mapped, err := unix.Mmap(int(f.Fd()), 0, int(fi.Size()), - unix.PROT_READ, unix.MAP_SHARED) - if err != nil { - f.Close() - return nil, err - } - - _, dir, err := packed.Load(f) - if err != nil { - unix.Munmap(mapped) - f.Close() - return nil, err - } - h := &Handler{ - f: f, - mapped: mapped, - dir: dir.Files, - headers: make(map[string]string), - startTime: time.Now(), + mapped: mapped.Data, + dir: mapped.Directory.Files, + headers: make(map[string]string), + cacheControl200: "no-cache", + cacheControl404: "no-cache", + startTime: time.Now(), } // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options - h.SetHeader("X-Frame-Options", "sameorigin") + h.SetHeader("X-Frame-Options", "SAMEORIGIN") // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options h.SetHeader("X-Content-Type-Options", "nosniff") @@ -67,11 +44,13 @@ func New(packfile string) (*Handler, error) { // Handler implements http.Handler and allows options to be set. type Handler struct { - f *os.File - mapped []byte - dir map[string]*packed.File - headers map[string]string - startTime time.Time + mapped []byte + dir map[string]*packed.File + headers map[string]string + cacheControl200 string + cacheControl404 string + startTime time.Time + notFound *packed.File } // SetHeader allows a custom header to be set on HTTP responses. These are @@ -110,6 +89,41 @@ func (h *Handler) SetIndex(filename string) { } } +// SetNotFound allows overriding the returned resource when a request is made +// for a resource that does not exist. The default behaviour would be to return +// a standard HTTP 404 Not Found response; calling this function with an empty +// string will restore that behaviour. +// +// This function will return an error if the named resource is not present in +// the packfile. +func (h *Handler) SetNotFound(notFound string) error { + if notFound == "" { + h.notFound = nil + return nil + } + + notFound = path.Clean(notFound) + dir := h.dir[path.Clean(notFound)] + if dir == nil { + return fmt.Errorf("no such resource %q", notFound) + } + h.notFound = dir + return nil +} + +// SetCacheControl allows control over the cache-control header. If not called, +// "Cache-Control: no-cache" is the default. +// +// The first and second arguments are required, and specify the header to use +// for 200 responses and 404 responses respectively. +// +// TODO: could be expanded to take extra varargs for more specific control over +// particular assets, like different expiry per content-type. +func (h *Handler) SetCacheControl(hdr200, hdr404 string) { + h.cacheControl200 = hdr200 + h.cacheControl404 = hdr404 +} + // ServeHTTP handles requests for files. It supports GET and HEAD methods, with // anything else returning a 405. Exact path matches are required, else a 404 is // returned. @@ -130,11 +144,16 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { info := h.dir[path.Clean(req.URL.Path)] if info == nil { - http.NotFound(w, req) - return + w.Header().Set("Cache-Control", h.cacheControl404) + if h.notFound == nil { + http.NotFound(w, req) + return + } + info = h.notFound } // set standard headers + w.Header().Set("Cache-Control", h.cacheControl200) w.Header().Set("Vary", "Accept-Encoding") w.Header().Set("Etag", info.Etag) w.Header().Set("Content-Type", info.ContentType) @@ -177,87 +196,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { if req.Method == "HEAD" { return } - h.sendfile(w, data, offset, length) -} -func (h *Handler) sendfile(w http.ResponseWriter, data *packed.FileData, - offset, length uint64, -) { - hj, ok := w.(http.Hijacker) - if !ok { - // fallback - h.copyfile(w, data, offset, length) - return - } - - conn, buf, err := hj.Hijack() - if err != nil { - // fallback - h.copyfile(w, data, offset, length) - return - } - - tcp, ok := conn.(*net.TCPConn) - if !ok { - // fallback - h.copyfile(w, data, offset, length) - return - } - defer tcp.Close() - - rawsock, err := tcp.SyscallConn() - if err == nil { - err = buf.Flush() - } - if err != nil { - // error only returned if the underlying connection is broken, - // so there's no point calling sendfile - return - } - - var breakErr error - off := int64(data.Offset + offset) - remain := length - - for breakErr == nil && remain > 0 { - // sendfile(2) can send a maximum of 1GiB - var amt int - if remain > (1 << 30) { - amt = (1 << 30) - } else { - amt = int(remain) - } - - // behaviour of control function: - // · some bytes written: sets written > 0, returns true (breaks - // out of loop on first write) - // · EAGAIN: returns false (causes Write() to loop until - // success or permanent failure) - // · other error: sets breakErr - var written int - rawsock.Write(func(outfd uintptr) bool { - written, err = unix.Sendfile(int(outfd), int(h.f.Fd()), &off, amt) - switch err { - case nil: - return true - case syscall.EAGAIN: - return false - default: - breakErr = err - return true - } - }) - - // we may have had a partial write, or file may have been > 1GiB - remain -= uint64(written) - } -} - -// copyfile is a fallback handler that uses write(2) on our memory-mapped data -// to push out the response. -func (h *Handler) copyfile(w http.ResponseWriter, data *packed.FileData, - offset, length uint64, -) { offset += data.Offset w.Write(h.mapped[offset : offset+length]) } diff --git a/packed/load.go b/packed/load.go index 10609e5..3dad308 100644 --- a/packed/load.go +++ b/packed/load.go @@ -2,9 +2,13 @@ package packed import ( fmt "fmt" + "io/fs" "os" "path" "strings" + + "golang.org/x/sys/unix" + "google.golang.org/protobuf/proto" ) const ( @@ -17,14 +21,14 @@ const ( VersionInitial = 1 ) -// Load a ready-packed file. -func Load(f *os.File) (*Header, *Directory, error) { - hdr, err := loadHeader(f) +// Read a ready-packed file. +func Read(f []byte) (*Header, *Directory, error) { + hdr, err := readHeader(f) if err != nil { return nil, nil, err } - dir, err := loadDirectory(f, hdr) + dir, err := readDirectory(f, hdr) if le, ok := err.(*LoadError); ok { // augment error le.Magic = hdr.Magic @@ -33,20 +37,101 @@ func Load(f *os.File) (*Header, *Directory, error) { return hdr, dir, err // we may have a partial dir } -// loadHeader retrieves and decodes the header from the start of the file. It -// ensures the magic number and the version number match. Errors are returned -// as type LoadError. -func loadHeader(f *os.File) (*Header, error) { - raw := make([]byte, 36) - if _, err := f.ReadAt(raw, 0); err != nil { - return nil, &LoadError{ - Cause: IOError, - Underlying: err, +// Mapped holds an mmap()ed packfile. +type Mapped struct { + Data []byte + Header *Header + Directory *Directory +} + +func (m *Mapped) Close() error { + return unix.Munmap(m.Data) +} + +// MapAndLoad mmaps the named file and reads the pack data. +func MapAndLoad(fname string) (*Mapped, error) { + f, err := os.Open(fname) + if err != nil { + return nil, err + } + defer f.Close() + + st, err := f.Stat() + if err != nil { + return nil, &fs.PathError{ + Path: fname, + Op: "stat", + Err: err, } } + data, err := unix.Mmap(int(f.Fd()), 0, int(st.Size()), unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + return nil, &fs.PathError{ + Path: fname, + Op: "mmap", + Err: err, + } + } + + hdr, dir, err := Read(data) + if err != nil { + unix.Munmap(data) + return nil, err + } + + return &Mapped{ + Data: data, + Header: hdr, + Directory: dir, + }, nil +} + +// Load a ready-packed file. +// +// Deprecated: use [Read] or [MapAndLoad]. +func Load(f *os.File) (*Header, *Directory, error) { + // we need to know the size of the file in order to mmap it + st, err := f.Stat() + if err != nil { + return nil, nil, &fs.PathError{ + Path: f.Name(), + Op: "stat", + Err: err, + } + } + + // compatibility: old code may have worked with non-zero offset + offset, err := f.Seek(0, os.SEEK_CUR) + if err != nil { + return nil, nil, &fs.PathError{ + Path: f.Name(), + Op: "seek", + Err: err, + } + } + + // mmap the file + data, err := unix.Mmap(int(f.Fd()), 0, int(st.Size()), unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + return nil, nil, &fs.PathError{ + Path: f.Name(), + Op: "mmap", + Err: err, + } + } + defer unix.Munmap(data) + + // read as though it's from memory + return Read(data[offset:]) +} + +// readHeader retrieves and decodes the header from the start of the file. It +// ensures the magic number and the version number match. Errors are returned +// as type LoadError. +func readHeader(f []byte) (*Header, error) { hdr := new(Header) - if err := hdr.Unmarshal(raw); err != nil { + if err := proto.Unmarshal(f[:36], hdr); err != nil { return nil, &LoadError{ Cause: HeaderUnmarshalError, Underlying: err, @@ -77,35 +162,22 @@ func loadHeader(f *os.File) (*Header, error) { return hdr, nil } -// loadDirectory reads the directory from a file. The directory is checked +// readDirectory reads the directory from a file. The directory is checked // for consistency (offsets, filenames) but not integrity (file data is not // read/checksummed). -func loadDirectory(f *os.File, hdr *Header) (*Directory, error) { - fi, err := f.Stat() - if err != nil { - return nil, &LoadError{ - Cause: IOError, - Underlying: err, - } - } - fileSize := uint64(fi.Size()) +func readDirectory(f []byte, hdr *Header) (*Directory, error) { + fileSize := uint64(len(f)) + start := hdr.DirectoryOffset + end := start + hdr.DirectoryLength - if hdr.DirectoryOffset+hdr.DirectoryLength > fileSize { + if end > fileSize { return nil, &LoadError{ Cause: BadOffsetError, } } - raw := make([]byte, hdr.DirectoryLength) - if _, err := f.ReadAt(raw, int64(hdr.DirectoryOffset)); err != nil { - return nil, &LoadError{ - Cause: IOError, - Underlying: err, - } - } - dir := new(Directory) - if err := dir.Unmarshal(raw); err != nil { + if err := proto.Unmarshal(f[start:end], dir); err != nil { return nil, &LoadError{ Cause: DirectoryUnmarshalError, Underlying: err, diff --git a/packed/load_test.go b/packed/load_test.go new file mode 100644 index 0000000..63be82e --- /dev/null +++ b/packed/load_test.go @@ -0,0 +1,39 @@ +package packed + +import ( + "bytes" + "os" + "testing" +) + +func TestMapAndLoad(t *testing.T) { + rawPack, err := os.ReadFile("testdata/helloworld.pack") + if err != nil { + t.Fatal(err) + } + rawText, err := os.ReadFile("testdata/helloworld.txt") + if err != nil { + t.Fatal(err) + } + + mapped, err := MapAndLoad("testdata/helloworld.pack") + if err != nil { + t.Fatal(err) + } + defer mapped.Close() + + if !bytes.Equal(mapped.Data, rawPack) { + t.Error("mapped data does not match raw file") + } + + file := mapped.Directory.Files["/helloworld.txt"] + if file == nil { + t.Fatal("directory does not contain /helloworld.txt") + } + + start := file.Uncompressed.Offset + end := start + file.Uncompressed.Length + if !bytes.Equal(rawText, mapped.Data[start:end]) { + t.Error("uncompressed file does not match testdata/helloworld.txt") + } +} diff --git a/packed/packed.pb.go b/packed/packed.pb.go index b551cd2..32fd75a 100644 --- a/packed/packed.pb.go +++ b/packed/packed.pb.go @@ -1,39 +1,32 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.21.6 // source: packed.proto -/* - Package packed is a generated protocol buffer package. - - It is generated from these files: - packed.proto - - It has these top-level messages: - Header - Directory - File - FileData -*/ package packed -import proto "github.com/gogo/protobuf/proto" -import fmt "fmt" -import math "math" +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) -import io "io" +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package - -// Header at start of file. This must be a fixed, known size. +// Header at start of file. This must be a fixed, known size. Fields cannot +// be zero. type Header struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Magic number, used to quickly detect misconfigured systems or // corrupted files. Magic uint64 `protobuf:"fixed64,1,opt,name=magic,proto3" json:"magic,omitempty"` @@ -47,60 +40,122 @@ type Header struct { DirectoryLength uint64 `protobuf:"fixed64,4,opt,name=directory_length,json=directoryLength,proto3" json:"directory_length,omitempty"` } -func (m *Header) Reset() { *m = Header{} } -func (m *Header) String() string { return proto.CompactTextString(m) } -func (*Header) ProtoMessage() {} -func (*Header) Descriptor() ([]byte, []int) { return fileDescriptorPacked, []int{0} } +func (x *Header) Reset() { + *x = Header{} + if protoimpl.UnsafeEnabled { + mi := &file_packed_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} -func (m *Header) GetMagic() uint64 { - if m != nil { - return m.Magic +func (x *Header) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Header) ProtoMessage() {} + +func (x *Header) ProtoReflect() protoreflect.Message { + mi := &file_packed_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Header.ProtoReflect.Descriptor instead. +func (*Header) Descriptor() ([]byte, []int) { + return file_packed_proto_rawDescGZIP(), []int{0} +} + +func (x *Header) GetMagic() uint64 { + if x != nil { + return x.Magic } return 0 } -func (m *Header) GetVersion() uint64 { - if m != nil { - return m.Version +func (x *Header) GetVersion() uint64 { + if x != nil { + return x.Version } return 0 } -func (m *Header) GetDirectoryOffset() uint64 { - if m != nil { - return m.DirectoryOffset +func (x *Header) GetDirectoryOffset() uint64 { + if x != nil { + return x.DirectoryOffset } return 0 } -func (m *Header) GetDirectoryLength() uint64 { - if m != nil { - return m.DirectoryLength +func (x *Header) GetDirectoryLength() uint64 { + if x != nil { + return x.DirectoryLength } return 0 } // Directory of available files. type Directory struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Files available within this pack. The key is the path of the URL to // serve, and the value describes the file associated with that path. - Files map[string]*File `protobuf:"bytes,1,rep,name=files" json:"files,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` + Files map[string]*File `protobuf:"bytes,1,rep,name=files,proto3" json:"files,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } -func (m *Directory) Reset() { *m = Directory{} } -func (m *Directory) String() string { return proto.CompactTextString(m) } -func (*Directory) ProtoMessage() {} -func (*Directory) Descriptor() ([]byte, []int) { return fileDescriptorPacked, []int{1} } +func (x *Directory) Reset() { + *x = Directory{} + if protoimpl.UnsafeEnabled { + mi := &file_packed_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} -func (m *Directory) GetFiles() map[string]*File { - if m != nil { - return m.Files +func (x *Directory) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Directory) ProtoMessage() {} + +func (x *Directory) ProtoReflect() protoreflect.Message { + mi := &file_packed_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Directory.ProtoReflect.Descriptor instead. +func (*Directory) Descriptor() ([]byte, []int) { + return file_packed_proto_rawDescGZIP(), []int{1} +} + +func (x *Directory) GetFiles() map[string]*File { + if x != nil { + return x.Files } return nil } // File that can be served. type File struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // ContentType of the file, copied directly into the "Content-Type" header. ContentType string `protobuf:"bytes,1,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"` // Etag of the file (includes double quotes). Remembered by the browser @@ -108,1086 +163,292 @@ type File struct { // requests. Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` // Uncompressed version of the file. - Uncompressed *FileData `protobuf:"bytes,3,opt,name=uncompressed" json:"uncompressed,omitempty"` + Uncompressed *FileData `protobuf:"bytes,3,opt,name=uncompressed,proto3" json:"uncompressed,omitempty"` // Gzip compressed version of the file. - Gzip *FileData `protobuf:"bytes,4,opt,name=gzip" json:"gzip,omitempty"` + Gzip *FileData `protobuf:"bytes,4,opt,name=gzip,proto3" json:"gzip,omitempty"` // Brotli compressed version of the file. - Brotli *FileData `protobuf:"bytes,5,opt,name=brotli" json:"brotli,omitempty"` + Brotli *FileData `protobuf:"bytes,5,opt,name=brotli,proto3" json:"brotli,omitempty"` } -func (m *File) Reset() { *m = File{} } -func (m *File) String() string { return proto.CompactTextString(m) } -func (*File) ProtoMessage() {} -func (*File) Descriptor() ([]byte, []int) { return fileDescriptorPacked, []int{2} } +func (x *File) Reset() { + *x = File{} + if protoimpl.UnsafeEnabled { + mi := &file_packed_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} -func (m *File) GetContentType() string { - if m != nil { - return m.ContentType +func (x *File) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*File) ProtoMessage() {} + +func (x *File) ProtoReflect() protoreflect.Message { + mi := &file_packed_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use File.ProtoReflect.Descriptor instead. +func (*File) Descriptor() ([]byte, []int) { + return file_packed_proto_rawDescGZIP(), []int{2} +} + +func (x *File) GetContentType() string { + if x != nil { + return x.ContentType } return "" } -func (m *File) GetEtag() string { - if m != nil { - return m.Etag +func (x *File) GetEtag() string { + if x != nil { + return x.Etag } return "" } -func (m *File) GetUncompressed() *FileData { - if m != nil { - return m.Uncompressed +func (x *File) GetUncompressed() *FileData { + if x != nil { + return x.Uncompressed } return nil } -func (m *File) GetGzip() *FileData { - if m != nil { - return m.Gzip +func (x *File) GetGzip() *FileData { + if x != nil { + return x.Gzip } return nil } -func (m *File) GetBrotli() *FileData { - if m != nil { - return m.Brotli +func (x *File) GetBrotli() *FileData { + if x != nil { + return x.Brotli } return nil } // FileData records the position of the file data within the pack. type FileData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Offset is the start of the file, in bytes relative to the start of // the pack. Offset uint64 `protobuf:"fixed64,1,opt,name=offset,proto3" json:"offset,omitempty"` - // Length is the + // Length of the (possibly compressed) file data, in bytes. Length uint64 `protobuf:"fixed64,2,opt,name=length,proto3" json:"length,omitempty"` } -func (m *FileData) Reset() { *m = FileData{} } -func (m *FileData) String() string { return proto.CompactTextString(m) } -func (*FileData) ProtoMessage() {} -func (*FileData) Descriptor() ([]byte, []int) { return fileDescriptorPacked, []int{3} } +func (x *FileData) Reset() { + *x = FileData{} + if protoimpl.UnsafeEnabled { + mi := &file_packed_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} -func (m *FileData) GetOffset() uint64 { - if m != nil { - return m.Offset +func (x *FileData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileData) ProtoMessage() {} + +func (x *FileData) ProtoReflect() protoreflect.Message { + mi := &file_packed_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileData.ProtoReflect.Descriptor instead. +func (*FileData) Descriptor() ([]byte, []int) { + return file_packed_proto_rawDescGZIP(), []int{3} +} + +func (x *FileData) GetOffset() uint64 { + if x != nil { + return x.Offset } return 0 } -func (m *FileData) GetLength() uint64 { - if m != nil { - return m.Length +func (x *FileData) GetLength() uint64 { + if x != nil { + return x.Length } return 0 } -func init() { - proto.RegisterType((*Header)(nil), "packed.Header") - proto.RegisterType((*Directory)(nil), "packed.Directory") - proto.RegisterType((*File)(nil), "packed.File") - proto.RegisterType((*FileData)(nil), "packed.FileData") -} -func (m *Header) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} +var File_packed_proto protoreflect.FileDescriptor -func (m *Header) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.Magic != 0 { - dAtA[i] = 0x9 - i++ - i = encodeFixed64Packed(dAtA, i, uint64(m.Magic)) - } - if m.Version != 0 { - dAtA[i] = 0x11 - i++ - i = encodeFixed64Packed(dAtA, i, uint64(m.Version)) - } - if m.DirectoryOffset != 0 { - dAtA[i] = 0x19 - i++ - i = encodeFixed64Packed(dAtA, i, uint64(m.DirectoryOffset)) - } - if m.DirectoryLength != 0 { - dAtA[i] = 0x21 - i++ - i = encodeFixed64Packed(dAtA, i, uint64(m.DirectoryLength)) - } - return i, nil -} - -func (m *Directory) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Directory) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Files) > 0 { - for k, _ := range m.Files { - dAtA[i] = 0xa - i++ - v := m.Files[k] - msgSize := 0 - if v != nil { - msgSize = v.Size() - msgSize += 1 + sovPacked(uint64(msgSize)) - } - mapSize := 1 + len(k) + sovPacked(uint64(len(k))) + msgSize - i = encodeVarintPacked(dAtA, i, uint64(mapSize)) - dAtA[i] = 0xa - i++ - i = encodeVarintPacked(dAtA, i, uint64(len(k))) - i += copy(dAtA[i:], k) - if v != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintPacked(dAtA, i, uint64(v.Size())) - n1, err := v.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n1 - } - } - } - return i, nil -} - -func (m *File) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *File) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.ContentType) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintPacked(dAtA, i, uint64(len(m.ContentType))) - i += copy(dAtA[i:], m.ContentType) - } - if len(m.Etag) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPacked(dAtA, i, uint64(len(m.Etag))) - i += copy(dAtA[i:], m.Etag) - } - if m.Uncompressed != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintPacked(dAtA, i, uint64(m.Uncompressed.Size())) - n2, err := m.Uncompressed.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n2 - } - if m.Gzip != nil { - dAtA[i] = 0x22 - i++ - i = encodeVarintPacked(dAtA, i, uint64(m.Gzip.Size())) - n3, err := m.Gzip.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n3 - } - if m.Brotli != nil { - dAtA[i] = 0x2a - i++ - i = encodeVarintPacked(dAtA, i, uint64(m.Brotli.Size())) - n4, err := m.Brotli.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n4 - } - return i, nil -} - -func (m *FileData) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *FileData) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.Offset != 0 { - dAtA[i] = 0x9 - i++ - i = encodeFixed64Packed(dAtA, i, uint64(m.Offset)) - } - if m.Length != 0 { - dAtA[i] = 0x11 - i++ - i = encodeFixed64Packed(dAtA, i, uint64(m.Length)) - } - return i, nil -} - -func encodeFixed64Packed(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Packed(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} -func encodeVarintPacked(dAtA []byte, offset int, v uint64) int { - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return offset + 1 -} -func (m *Header) Size() (n int) { - var l int - _ = l - if m.Magic != 0 { - n += 9 - } - if m.Version != 0 { - n += 9 - } - if m.DirectoryOffset != 0 { - n += 9 - } - if m.DirectoryLength != 0 { - n += 9 - } - return n -} - -func (m *Directory) Size() (n int) { - var l int - _ = l - if len(m.Files) > 0 { - for k, v := range m.Files { - _ = k - _ = v - l = 0 - if v != nil { - l = v.Size() - l += 1 + sovPacked(uint64(l)) - } - mapEntrySize := 1 + len(k) + sovPacked(uint64(len(k))) + l - n += mapEntrySize + 1 + sovPacked(uint64(mapEntrySize)) - } - } - return n -} - -func (m *File) Size() (n int) { - var l int - _ = l - l = len(m.ContentType) - if l > 0 { - n += 1 + l + sovPacked(uint64(l)) - } - l = len(m.Etag) - if l > 0 { - n += 1 + l + sovPacked(uint64(l)) - } - if m.Uncompressed != nil { - l = m.Uncompressed.Size() - n += 1 + l + sovPacked(uint64(l)) - } - if m.Gzip != nil { - l = m.Gzip.Size() - n += 1 + l + sovPacked(uint64(l)) - } - if m.Brotli != nil { - l = m.Brotli.Size() - n += 1 + l + sovPacked(uint64(l)) - } - return n -} - -func (m *FileData) Size() (n int) { - var l int - _ = l - if m.Offset != 0 { - n += 9 - } - if m.Length != 0 { - n += 9 - } - return n -} - -func sovPacked(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n -} -func sozPacked(x uint64) (n int) { - return sovPacked(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *Header) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Header: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Header: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 1 { - return fmt.Errorf("proto: wrong wireType = %d for field Magic", wireType) - } - m.Magic = 0 - if (iNdEx + 8) > l { - return io.ErrUnexpectedEOF - } - iNdEx += 8 - m.Magic = uint64(dAtA[iNdEx-8]) - m.Magic |= uint64(dAtA[iNdEx-7]) << 8 - m.Magic |= uint64(dAtA[iNdEx-6]) << 16 - m.Magic |= uint64(dAtA[iNdEx-5]) << 24 - m.Magic |= uint64(dAtA[iNdEx-4]) << 32 - m.Magic |= uint64(dAtA[iNdEx-3]) << 40 - m.Magic |= uint64(dAtA[iNdEx-2]) << 48 - m.Magic |= uint64(dAtA[iNdEx-1]) << 56 - case 2: - if wireType != 1 { - return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) - } - m.Version = 0 - if (iNdEx + 8) > l { - return io.ErrUnexpectedEOF - } - iNdEx += 8 - m.Version = uint64(dAtA[iNdEx-8]) - m.Version |= uint64(dAtA[iNdEx-7]) << 8 - m.Version |= uint64(dAtA[iNdEx-6]) << 16 - m.Version |= uint64(dAtA[iNdEx-5]) << 24 - m.Version |= uint64(dAtA[iNdEx-4]) << 32 - m.Version |= uint64(dAtA[iNdEx-3]) << 40 - m.Version |= uint64(dAtA[iNdEx-2]) << 48 - m.Version |= uint64(dAtA[iNdEx-1]) << 56 - case 3: - if wireType != 1 { - return fmt.Errorf("proto: wrong wireType = %d for field DirectoryOffset", wireType) - } - m.DirectoryOffset = 0 - if (iNdEx + 8) > l { - return io.ErrUnexpectedEOF - } - iNdEx += 8 - m.DirectoryOffset = uint64(dAtA[iNdEx-8]) - m.DirectoryOffset |= uint64(dAtA[iNdEx-7]) << 8 - m.DirectoryOffset |= uint64(dAtA[iNdEx-6]) << 16 - m.DirectoryOffset |= uint64(dAtA[iNdEx-5]) << 24 - m.DirectoryOffset |= uint64(dAtA[iNdEx-4]) << 32 - m.DirectoryOffset |= uint64(dAtA[iNdEx-3]) << 40 - m.DirectoryOffset |= uint64(dAtA[iNdEx-2]) << 48 - m.DirectoryOffset |= uint64(dAtA[iNdEx-1]) << 56 - case 4: - if wireType != 1 { - return fmt.Errorf("proto: wrong wireType = %d for field DirectoryLength", wireType) - } - m.DirectoryLength = 0 - if (iNdEx + 8) > l { - return io.ErrUnexpectedEOF - } - iNdEx += 8 - m.DirectoryLength = uint64(dAtA[iNdEx-8]) - m.DirectoryLength |= uint64(dAtA[iNdEx-7]) << 8 - m.DirectoryLength |= uint64(dAtA[iNdEx-6]) << 16 - m.DirectoryLength |= uint64(dAtA[iNdEx-5]) << 24 - m.DirectoryLength |= uint64(dAtA[iNdEx-4]) << 32 - m.DirectoryLength |= uint64(dAtA[iNdEx-3]) << 40 - m.DirectoryLength |= uint64(dAtA[iNdEx-2]) << 48 - m.DirectoryLength |= uint64(dAtA[iNdEx-1]) << 56 - default: - iNdEx = preIndex - skippy, err := skipPacked(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPacked - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Directory) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Directory: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Directory: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Files", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPacked - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Files == nil { - m.Files = make(map[string]*File) - } - var mapkey string - var mapvalue *File - for iNdEx < postIndex { - entryPreIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - if fieldNum == 1 { - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPacked - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey - } else if fieldNum == 2 { - var mapmsglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - mapmsglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if mapmsglen < 0 { - return ErrInvalidLengthPacked - } - postmsgIndex := iNdEx + mapmsglen - if mapmsglen < 0 { - return ErrInvalidLengthPacked - } - if postmsgIndex > l { - return io.ErrUnexpectedEOF - } - mapvalue = &File{} - if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { - return err - } - iNdEx = postmsgIndex - } else { - iNdEx = entryPreIndex - skippy, err := skipPacked(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPacked - } - if (iNdEx + skippy) > postIndex { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - m.Files[mapkey] = mapvalue - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPacked(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPacked - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *File) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: File: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: File: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ContentType", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPacked - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ContentType = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Etag", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPacked - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Etag = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Uncompressed", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPacked - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Uncompressed == nil { - m.Uncompressed = &FileData{} - } - if err := m.Uncompressed.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Gzip", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPacked - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Gzip == nil { - m.Gzip = &FileData{} - } - if err := m.Gzip.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 5: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Brotli", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPacked - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Brotli == nil { - m.Brotli = &FileData{} - } - if err := m.Brotli.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPacked(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPacked - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *FileData) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPacked - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: FileData: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: FileData: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 1 { - return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) - } - m.Offset = 0 - if (iNdEx + 8) > l { - return io.ErrUnexpectedEOF - } - iNdEx += 8 - m.Offset = uint64(dAtA[iNdEx-8]) - m.Offset |= uint64(dAtA[iNdEx-7]) << 8 - m.Offset |= uint64(dAtA[iNdEx-6]) << 16 - m.Offset |= uint64(dAtA[iNdEx-5]) << 24 - m.Offset |= uint64(dAtA[iNdEx-4]) << 32 - m.Offset |= uint64(dAtA[iNdEx-3]) << 40 - m.Offset |= uint64(dAtA[iNdEx-2]) << 48 - m.Offset |= uint64(dAtA[iNdEx-1]) << 56 - case 2: - if wireType != 1 { - return fmt.Errorf("proto: wrong wireType = %d for field Length", wireType) - } - m.Length = 0 - if (iNdEx + 8) > l { - return io.ErrUnexpectedEOF - } - iNdEx += 8 - m.Length = uint64(dAtA[iNdEx-8]) - m.Length |= uint64(dAtA[iNdEx-7]) << 8 - m.Length |= uint64(dAtA[iNdEx-6]) << 16 - m.Length |= uint64(dAtA[iNdEx-5]) << 24 - m.Length |= uint64(dAtA[iNdEx-4]) << 32 - m.Length |= uint64(dAtA[iNdEx-3]) << 40 - m.Length |= uint64(dAtA[iNdEx-2]) << 48 - m.Length |= uint64(dAtA[iNdEx-1]) << 56 - default: - iNdEx = preIndex - skippy, err := skipPacked(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPacked - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipPacked(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowPacked - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowPacked - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowPacked - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - iNdEx += length - if length < 0 { - return 0, ErrInvalidLengthPacked - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowPacked - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipPacked(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") +var file_packed_proto_rawDesc = []byte{ + 0x0a, 0x0c, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, + 0x6c, 0x77, 0x69, 0x74, 0x68, 0x65, 0x72, 0x73, 0x2e, 0x68, 0x74, 0x70, 0x61, 0x63, 0x6b, 0x2e, + 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x22, 0x8e, 0x01, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, 0x67, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x06, + 0x52, 0x05, 0x6d, 0x61, 0x67, 0x69, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x29, 0x0a, 0x10, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x6f, + 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0f, 0x64, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x29, 0x0a, 0x10, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x06, 0x52, 0x0f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, + 0x79, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0xa7, 0x01, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, + 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x42, 0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x6c, 0x77, 0x69, 0x74, 0x68, 0x65, 0x72, 0x73, 0x2e, + 0x68, 0x74, 0x70, 0x61, 0x63, 0x6b, 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x2e, 0x44, 0x69, + 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x1a, 0x56, 0x0a, 0x0a, 0x46, 0x69, 0x6c, + 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6c, 0x77, 0x69, 0x74, 0x68, + 0x65, 0x72, 0x73, 0x2e, 0x68, 0x74, 0x70, 0x61, 0x63, 0x6b, 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, + 0x64, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0xf3, 0x01, 0x0a, 0x04, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x65, 0x74, 0x61, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x65, 0x74, 0x61, + 0x67, 0x12, 0x44, 0x0a, 0x0c, 0x75, 0x6e, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6c, 0x77, 0x69, 0x74, 0x68, 0x65, + 0x72, 0x73, 0x2e, 0x68, 0x74, 0x70, 0x61, 0x63, 0x6b, 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, + 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x52, 0x0c, 0x75, 0x6e, 0x63, 0x6f, 0x6d, + 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x12, 0x34, 0x0a, 0x04, 0x67, 0x7a, 0x69, 0x70, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6c, 0x77, 0x69, 0x74, 0x68, 0x65, 0x72, 0x73, + 0x2e, 0x68, 0x74, 0x70, 0x61, 0x63, 0x6b, 0x2e, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x2e, 0x46, + 0x69, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x67, 0x7a, 0x69, 0x70, 0x12, 0x38, 0x0a, + 0x06, 0x62, 0x72, 0x6f, 0x74, 0x6c, 0x69, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, + 0x6c, 0x77, 0x69, 0x74, 0x68, 0x65, 0x72, 0x73, 0x2e, 0x68, 0x74, 0x70, 0x61, 0x63, 0x6b, 0x2e, + 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x52, + 0x06, 0x62, 0x72, 0x6f, 0x74, 0x6c, 0x69, 0x22, 0x3a, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x44, + 0x61, 0x74, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x06, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6c, + 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x06, 0x52, 0x06, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x42, 0x22, 0x5a, 0x20, 0x73, 0x72, 0x63, 0x2e, 0x6c, 0x77, 0x69, 0x74, 0x68, + 0x65, 0x72, 0x73, 0x2e, 0x6d, 0x65, 0x2e, 0x75, 0x6b, 0x2f, 0x68, 0x74, 0x70, 0x61, 0x63, 0x6b, + 0x2f, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( - ErrInvalidLengthPacked = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowPacked = fmt.Errorf("proto: integer overflow") + file_packed_proto_rawDescOnce sync.Once + file_packed_proto_rawDescData = file_packed_proto_rawDesc ) -func init() { proto.RegisterFile("packed.proto", fileDescriptorPacked) } - -var fileDescriptorPacked = []byte{ - // 359 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x74, 0x92, 0xc1, 0x4e, 0xab, 0x50, - 0x10, 0x86, 0x2f, 0x2d, 0x70, 0x6f, 0x07, 0x92, 0x4b, 0x26, 0xc6, 0x10, 0x63, 0x1a, 0x25, 0x2e, - 0xea, 0xa6, 0x0b, 0x74, 0x61, 0xba, 0x34, 0xb5, 0x71, 0x61, 0x62, 0x42, 0xdc, 0x37, 0x14, 0xa6, - 0x48, 0x4a, 0x81, 0x1c, 0x4e, 0x9b, 0xe0, 0x0b, 0xb8, 0x33, 0xbe, 0x93, 0x1b, 0x97, 0x3e, 0x82, - 0xd1, 0x17, 0x11, 0x0e, 0x87, 0x6a, 0x8d, 0x2e, 0x4e, 0x72, 0xfe, 0x6f, 0x7e, 0x06, 0x66, 0x7e, - 0xc0, 0xcc, 0xfd, 0x60, 0x41, 0xe1, 0x30, 0x67, 0x19, 0xcf, 0x50, 0x6f, 0x94, 0xf3, 0xa0, 0x80, - 0x7e, 0x49, 0x7e, 0x48, 0x0c, 0x77, 0x40, 0x5b, 0xfa, 0x51, 0x1c, 0xd8, 0xca, 0x81, 0x32, 0xd0, - 0xbd, 0x46, 0xa0, 0x0d, 0x7f, 0xd7, 0xc4, 0x8a, 0x38, 0x4b, 0xed, 0x8e, 0xe0, 0xad, 0xc4, 0x63, - 0xb0, 0xc2, 0x98, 0x51, 0xc0, 0x33, 0x56, 0x4e, 0xb3, 0xf9, 0xbc, 0x20, 0x6e, 0x77, 0x85, 0xe5, - 0xff, 0x86, 0x5f, 0x0b, 0xbc, 0x6d, 0x4d, 0x28, 0x8d, 0xf8, 0xad, 0xad, 0x7e, 0xb3, 0x5e, 0x09, - 0xec, 0xdc, 0x2b, 0xd0, 0x1b, 0xb7, 0x0c, 0x5d, 0xd0, 0xe6, 0x71, 0x42, 0x45, 0xf5, 0x4d, 0xdd, - 0x81, 0xe1, 0xee, 0x0f, 0xe5, 0x10, 0x1b, 0xc7, 0x70, 0x52, 0x97, 0x2f, 0x52, 0xce, 0x4a, 0xaf, - 0xb1, 0xee, 0x4d, 0x00, 0x3e, 0x21, 0x5a, 0xd0, 0x5d, 0x50, 0x29, 0x66, 0xea, 0x79, 0xf5, 0x15, - 0x1d, 0xd0, 0xd6, 0x7e, 0xb2, 0x22, 0x31, 0x8f, 0xe1, 0x9a, 0x6d, 0xcf, 0xfa, 0x21, 0xaf, 0x29, - 0x8d, 0x3a, 0x67, 0x8a, 0xf3, 0xa4, 0x80, 0x5a, 0x33, 0x3c, 0x04, 0x33, 0xc8, 0x52, 0x4e, 0x29, - 0x9f, 0xf2, 0x32, 0x27, 0xd9, 0xcb, 0x90, 0xec, 0xa6, 0x42, 0x88, 0xa0, 0x12, 0xf7, 0x23, 0xd1, - 0xb2, 0xe7, 0x89, 0x3b, 0x9e, 0x82, 0xb9, 0x4a, 0x83, 0x6c, 0x99, 0x33, 0x2a, 0x0a, 0x0a, 0xc5, - 0x6e, 0x0c, 0xd7, 0xfa, 0xfa, 0xba, 0xb1, 0xcf, 0x7d, 0x6f, 0xcb, 0x85, 0x47, 0xa0, 0x46, 0x77, - 0x71, 0x2e, 0xd6, 0xf3, 0x93, 0x5b, 0x54, 0x71, 0x00, 0xfa, 0xac, 0xca, 0x31, 0x89, 0x6d, 0xed, - 0x17, 0x9f, 0xac, 0x3b, 0x23, 0xf8, 0xd7, 0x32, 0xdc, 0x05, 0x5d, 0xe6, 0xd4, 0x44, 0x2c, 0x55, - 0xcd, 0x65, 0x28, 0x4d, 0xc4, 0x52, 0x9d, 0x5b, 0xcf, 0x6f, 0x7d, 0xe5, 0xa5, 0x3a, 0xaf, 0xd5, - 0x79, 0x7c, 0xef, 0xff, 0x99, 0xe9, 0xe2, 0xef, 0x39, 0xf9, 0x08, 0x00, 0x00, 0xff, 0xff, 0x07, - 0x4a, 0x55, 0x53, 0x4d, 0x02, 0x00, 0x00, +func file_packed_proto_rawDescGZIP() []byte { + file_packed_proto_rawDescOnce.Do(func() { + file_packed_proto_rawDescData = protoimpl.X.CompressGZIP(file_packed_proto_rawDescData) + }) + return file_packed_proto_rawDescData +} + +var file_packed_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_packed_proto_goTypes = []interface{}{ + (*Header)(nil), // 0: lwithers.htpack.packed.Header + (*Directory)(nil), // 1: lwithers.htpack.packed.Directory + (*File)(nil), // 2: lwithers.htpack.packed.File + (*FileData)(nil), // 3: lwithers.htpack.packed.FileData + nil, // 4: lwithers.htpack.packed.Directory.FilesEntry +} +var file_packed_proto_depIdxs = []int32{ + 4, // 0: lwithers.htpack.packed.Directory.files:type_name -> lwithers.htpack.packed.Directory.FilesEntry + 3, // 1: lwithers.htpack.packed.File.uncompressed:type_name -> lwithers.htpack.packed.FileData + 3, // 2: lwithers.htpack.packed.File.gzip:type_name -> lwithers.htpack.packed.FileData + 3, // 3: lwithers.htpack.packed.File.brotli:type_name -> lwithers.htpack.packed.FileData + 2, // 4: lwithers.htpack.packed.Directory.FilesEntry.value:type_name -> lwithers.htpack.packed.File + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_packed_proto_init() } +func file_packed_proto_init() { + if File_packed_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_packed_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Header); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_packed_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Directory); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_packed_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*File); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_packed_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_packed_proto_rawDesc, + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_packed_proto_goTypes, + DependencyIndexes: file_packed_proto_depIdxs, + MessageInfos: file_packed_proto_msgTypes, + }.Build() + File_packed_proto = out.File + file_packed_proto_rawDesc = nil + file_packed_proto_goTypes = nil + file_packed_proto_depIdxs = nil } diff --git a/packed/packed.proto b/packed/packed.proto index e797e59..771e1ec 100644 --- a/packed/packed.proto +++ b/packed/packed.proto @@ -1,6 +1,7 @@ syntax = "proto3"; -package packed; +package lwithers.htpack.packed; +option go_package = "src.lwithers.me.uk/htpack/packed"; // Header at start of file. This must be a fixed, known size. Fields cannot // be zero. @@ -54,6 +55,6 @@ message FileData { // the pack. fixed64 offset = 1; - // Length is the + // Length of the (possibly compressed) file data, in bytes. fixed64 length = 2; } diff --git a/packed/testdata/helloworld.pack b/packed/testdata/helloworld.pack new file mode 100644 index 0000000..8d8541e Binary files /dev/null and b/packed/testdata/helloworld.pack differ diff --git a/packed/testdata/helloworld.txt b/packed/testdata/helloworld.txt new file mode 100644 index 0000000..3898fd6 --- /dev/null +++ b/packed/testdata/helloworld.txt @@ -0,0 +1,13 @@ +Hello, world + +This file contains a few lines of plain text, really just enough +to trigger the "compression worthwhile" heuristic. + +This file contains a few lines of plain text, really just enough +to trigger the "compression worthwhile" heuristic. + +This file contains a few lines of plain text, really just enough +to trigger the "compression worthwhile" heuristic. + +This file contains a few lines of plain text, really just enough +to trigger the "compression worthwhile" heuristic.