Single range support

We now support single part ranges, as per:

https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests#Single_part_ranges

Multi-part ranges are not implemented because there have been far too
many bugs in this area.

It interacts with compression by selecting a byte range from the
compressed stream. Since the compressed stream is fixed, the results are
consistent.
This commit is contained in:
Laurence Withers 2019-10-09 13:04:07 +01:00
parent 8fc082c4ca
commit f7cd22f633
2 changed files with 84 additions and 13 deletions

View File

@ -10,7 +10,7 @@ support for:
- brotli, if you have the external compression binary available at pack time - brotli, if you have the external compression binary available at pack time
- does not yet support Transfer-Encoding, only Accept-Encoding/Content-Encoding - does not yet support Transfer-Encoding, only Accept-Encoding/Content-Encoding
- etags - etags
- ranges (TODO) - ranges
The workflow is as follows: The workflow is as follows:
- (optional) build YAML file describing files to serve - (optional) build YAML file describing files to serve
@ -18,3 +18,13 @@ The workflow is as follows:
- create `htpack.Handler` pointing at .htpack file - create `htpack.Handler` pointing at .htpack file
The handler can easily be combined with middleware (`http.StripPrefix` etc.). The handler can easily be combined with middleware (`http.StripPrefix` etc.).
## Range handling notes
Too many bugs have been found with range handling and composite ranges, so the
handler only accepts a single range within the limits of the file. Anything
else will be ignored.
The interaction between range handling and compression also seems a little
ill-defined; as we have pre-compressed data, however, we can consistently
serve the exact same byte data for compressed files.

View File

@ -1,6 +1,7 @@
package htpack package htpack
import ( import (
"fmt"
"net" "net"
"net/http" "net/http"
"os" "os"
@ -137,6 +138,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Vary", "Accept-Encoding") w.Header().Set("Vary", "Accept-Encoding")
w.Header().Set("Etag", info.Etag) w.Header().Set("Etag", info.Etag)
w.Header().Set("Content-Type", info.ContentType) w.Header().Set("Content-Type", info.ContentType)
w.Header().Set("Accept-Ranges", "bytes")
// process etag / modtime // process etag / modtime
if clientHasCachedVersion(info.Etag, h.startTime, req) { if clientHasCachedVersion(info.Etag, h.startTime, req) {
@ -155,38 +157,50 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Encoding", encodingGzip) w.Header().Set("Content-Encoding", encodingGzip)
} }
// TODO: Range // range support (single-part ranges only)
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests#Single_part_ranges
offset, length, isPartial := getFileRange(data, req)
if isPartial {
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d",
offset, offset+length-1, data.Length))
}
// now we know exactly what we're writing, finalise HTTP header // now we know exactly what we're writing, finalise HTTP header
w.Header().Set("Content-Length", strconv.FormatUint(data.Length, 10)) w.Header().Set("Content-Length", strconv.FormatUint(length, 10))
w.WriteHeader(http.StatusOK) if isPartial {
w.WriteHeader(http.StatusPartialContent)
} else {
w.WriteHeader(http.StatusOK)
}
// send body (though not for HEAD) // send body (though not for HEAD)
if req.Method == "HEAD" { if req.Method == "HEAD" {
return return
} }
h.sendfile(w, data) h.sendfile(w, data, offset, length)
} }
func (h *Handler) sendfile(w http.ResponseWriter, data *packed.FileData) { func (h *Handler) sendfile(w http.ResponseWriter, data *packed.FileData,
offset, length uint64,
) {
hj, ok := w.(http.Hijacker) hj, ok := w.(http.Hijacker)
if !ok { if !ok {
// fallback // fallback
h.copyfile(w, data) h.copyfile(w, data, offset, length)
return return
} }
conn, buf, err := hj.Hijack() conn, buf, err := hj.Hijack()
if err != nil { if err != nil {
// fallback // fallback
h.copyfile(w, data) h.copyfile(w, data, offset, length)
return return
} }
tcp, ok := conn.(*net.TCPConn) tcp, ok := conn.(*net.TCPConn)
if !ok { if !ok {
// fallback // fallback
h.copyfile(w, data) h.copyfile(w, data, offset, length)
return return
} }
defer tcp.Close() defer tcp.Close()
@ -202,8 +216,8 @@ func (h *Handler) sendfile(w http.ResponseWriter, data *packed.FileData) {
} }
var breakErr error var breakErr error
off := int64(data.Offset) off := int64(data.Offset + offset)
remain := data.Length remain := length
for breakErr == nil && remain > 0 { for breakErr == nil && remain > 0 {
// sendfile(2) can send a maximum of 1GiB // sendfile(2) can send a maximum of 1GiB
@ -239,8 +253,13 @@ func (h *Handler) sendfile(w http.ResponseWriter, data *packed.FileData) {
} }
} }
func (h *Handler) copyfile(w http.ResponseWriter, data *packed.FileData) { // copyfile is a fallback handler that uses write(2) on our memory-mapped data
w.Write(h.mapped[data.Offset : data.Offset+data.Length]) // to push out the response.
func (h *Handler) copyfile(w http.ResponseWriter, data *packed.FileData,
offset, length uint64,
) {
offset += data.Offset
w.Write(h.mapped[offset : offset+length])
} }
func acceptedEncodings(req *http.Request) (gzip, brotli bool) { func acceptedEncodings(req *http.Request) (gzip, brotli bool) {
@ -283,3 +302,45 @@ func clientHasCachedVersion(etag string, startTime time.Time, req *http.Request,
} }
return cachedTime.After(startTime) return cachedTime.After(startTime)
} }
// getFileRange returns the byte offset and length of the file to serve, along
// with whether or not it's partial content.
func getFileRange(data *packed.FileData, req *http.Request) (offset, length uint64, isPartial bool) {
length = data.Length
// only accept "Range: bytes=…"
r := req.Header.Get("Range")
if !strings.HasPrefix(r, "bytes=") {
return
}
r = strings.TrimPrefix(r, "bytes=")
// only accept a single range, "from-to", mapping to interval [from,to]
pos := strings.IndexByte(r, '-')
if pos == -1 {
return
}
sfrom, sto := r[:pos], r[pos+1:]
from, err := strconv.ParseUint(sfrom, 10, 64)
if err != nil {
return
}
to, err := strconv.ParseUint(sto, 10, 64)
if err != nil {
return
}
// validate the interval lies within the file
switch {
case from > to,
from >= data.Length,
to >= data.Length:
return
}
// all good
offset = from
length = to - from + 1
isPartial = true
return
}