diff --git a/pkg/analyzer/LICENSE.phash b/pkg/analyzer/LICENSE.phash new file mode 100644 index 00000000..eed63af2 --- /dev/null +++ b/pkg/analyzer/LICENSE.phash @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Adrien Delorme + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pkg/analyzer/image.go b/pkg/analyzer/image.go index 67267dfd..68650049 100644 --- a/pkg/analyzer/image.go +++ b/pkg/analyzer/image.go @@ -268,7 +268,7 @@ func InspectImage(system fs.FS, link manifest.Link, algorithms []manifest.HashAl if slices.Contains(neededAlgorithms, manifest.HashAlgorithmPhashDCT) { // Create phash and put it in a byte array p.Hashes.PhashDCT = make([]byte, 8) - binary.BigEndian.PutUint64(p.Hashes.PhashDCT, phash.DTC(img)) + binary.BigEndian.PutUint64(p.Hashes.PhashDCT, perceptualHash(img)) } if slices.Contains(neededAlgorithms, blurHashAlgorithm) { // Create the blurhash diff --git a/pkg/analyzer/image_bench_test.go b/pkg/analyzer/image_bench_test.go new file mode 100644 index 00000000..84d106a9 --- /dev/null +++ b/pkg/analyzer/image_bench_test.go @@ -0,0 +1,155 @@ +package analyzer + +import ( + "bytes" + "image" + "os" + "testing" + "testing/fstest" + + "github.com/azr/phash" + "github.com/bbrks/go-blurhash" + "github.com/disintegration/imaging" + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/readium/go-toolkit/pkg/mediatype" +) + +var benchmarkHashImplementations = []struct { + name string + hash func(image.Image) uint64 +}{ + {"Reference", phash.DTC}, + {"Optimized", perceptualHash}, +} + +func benchmarkImageFixture(b *testing.B) ([]byte, image.Image) { + b.Helper() + data, err := os.ReadFile("testdata/catsink.jpg") + if err != nil { + b.Fatal(err) + } + img, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + b.Fatal(err) + } + return data, img +} + +// BenchmarkImageAnalysis isolates each stage. The hash comparison includes its +// internal 32x32 resize; fixture decoding and the initial 128-pixel resize happen +// before timing. Reference uses the pinned phash dependency's original full DCT. +func BenchmarkImageAnalysis(b *testing.B) { + data, img := benchmarkImageFixture(b) + thumbnail := imaging.Resize(img, 128, 0, imaging.Lanczos) + b.Run("DecodeJPEG", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, _, err := image.Decode(bytes.NewReader(data)); err != nil { + b.Fatal(err) + } + } + }) + b.Run("Resize128", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + imaging.Resize(img, 128, 0, imaging.Lanczos) + } + }) + b.Run("PhashDCT", func(b *testing.B) { + want := phash.DTC(thumbnail) + for _, impl := range benchmarkHashImplementations { + b.Run(impl.name, func(b *testing.B) { + b.ReportAllocs() + var hash uint64 + for b.Loop() { + hash = impl.hash(thumbnail) + } + if hash != want { + b.Fatalf("hash = %016x, reference = %016x", hash, want) + } + }) + } + }) + b.Run("BlurHash5x5", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, err := blurhash.Encode(5, 5, thumbnail); err != nil { + b.Fatal(err) + } + } + }) +} + +// BenchmarkVisualHashPipeline compares the original and optimized hash within +// the same decode/resize/hash pipeline. Every iteration decodes the JPEG, resizes +// it to 128 pixels, and computes the selected hashes. File I/O and manifest work +// are excluded; BenchmarkInspectImage below measures the full current API. +func BenchmarkVisualHashPipeline(b *testing.B) { + data, img := benchmarkImageFixture(b) + thumbnail := imaging.Resize(img, 128, 0, imaging.Lanczos) + wantHash := phash.DTC(thumbnail) + wantBlur, err := blurhash.Encode(5, 5, thumbnail) + if err != nil { + b.Fatal(err) + } + for _, mode := range []string{"Phash", "Both"} { + b.Run(mode, func(b *testing.B) { + for _, impl := range benchmarkHashImplementations { + b.Run(impl.name, func(b *testing.B) { + b.ReportAllocs() + var hash uint64 + var blur string + for b.Loop() { + decoded, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + b.Fatal(err) + } + resized := imaging.Resize(decoded, 128, 0, imaging.Lanczos) + hash = impl.hash(resized) + if mode == "Both" { + blur, err = blurhash.Encode(5, 5, resized) + if err != nil { + b.Fatal(err) + } + } + } + if hash != wantHash || (mode == "Both" && blur != wantBlur) { + b.Fatal("hashes differ from the reference pipeline") + } + }) + } + }) + } +} + +// BenchmarkInspectImage measures the current public API using an in-memory file +// system to keep disk latency out of the result. It includes image decoding, +// resizing, visual hashing, and manifest property construction. +func BenchmarkInspectImage(b *testing.B) { + data, err := os.ReadFile("testdata/catsink.jpg") + if err != nil { + b.Fatal(err) + } + fs := fstest.MapFS{"catsink.jpg": &fstest.MapFile{Data: data}} + link := manifest.Link{ + Href: manifest.MustNewHREFFromString("catsink.jpg", false), + MediaType: &mediatype.JPEG, + } + for _, tc := range []struct { + name string + algorithms []manifest.HashAlgorithm + }{ + {"Phash", []manifest.HashAlgorithm{manifest.HashAlgorithmPhashDCT}}, + {"BlurHash", []manifest.HashAlgorithm{blurHashAlgorithm}}, + {"Both", []manifest.HashAlgorithm{manifest.HashAlgorithmPhashDCT, blurHashAlgorithm}}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, err := InspectImage(fs, link, tc.algorithms); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/analyzer/phash.go b/pkg/analyzer/phash.go new file mode 100644 index 00000000..bf3723e3 --- /dev/null +++ b/pkg/analyzer/phash.go @@ -0,0 +1,72 @@ +package analyzer + +// Adapted from github.com/azr/phash v0.2.0, copyright (c) 2020 Adrien Delorme. +// See LICENSE.phash for the MIT license. + +import ( + "image" + "math" + "slices" + + "github.com/disintegration/imaging" +) + +const phashSize = 32 +const phashFrequencies = 8 + +// The reference hash uses frequencies 1 through 8 on both axes. Cache their +// cosine values, preserving the reference expression's floating-point order. +var phashCosines = func() [phashFrequencies][phashSize]float64 { + var cosines [phashFrequencies][phashSize]float64 + for frequency := range cosines { + for sample := range cosines[frequency] { + index := (2*sample + 1) * (frequency + 1) + cosines[frequency][sample] = math.Cos(float64(index) / float64(2*phashSize) * math.Pi) + } + } + return cosines +}() + +// perceptualHash produces the same hash as phash.DTC, computing only the 64 +// coefficients it retains instead of all 1024. Keep the pixel traversal and +// summation order: reassociating the DCT can change bits near the median. +func perceptualHash(img image.Image) uint64 { + if img == nil { + return 0 + } + thumbnail := imaging.Resize(img, phashSize, phashSize, imaging.Lanczos) + var pixels [phashSize * phashSize]float64 + for x := range phashSize { + for y := range phashSize { + // NRGBAAt avoids boxing each pixel as color.Color. RGBA preserves + // the reference's 16-bit, alpha-premultiplied color conversion. + r, g, b, _ := thumbnail.NRGBAAt(x, y).RGBA() + pixels[x*phashSize+y] = 0.299*float64(r) + 0.587*float64(g) + 0.114*float64(b) + } + } + + var coefficients [phashFrequencies * phashFrequencies]float64 + for u := range phashFrequencies { + for v := range phashFrequencies { + var sum float64 + for x := range phashSize { + for y := range phashSize { + sum += phashCosines[u][x] * phashCosines[v][y] * pixels[x*phashSize+y] + } + } + // The normalization coefficients are both 1 for nonzero frequencies. + coefficients[u*phashFrequencies+v] = sum * 0.25 + } + } + + sorted := coefficients + slices.Sort(sorted[:]) + median := sorted[len(sorted)/2] + var hash uint64 + for bit, coefficient := range coefficients { + if coefficient > median { + hash |= uint64(1) << bit + } + } + return hash +} diff --git a/pkg/analyzer/phash_test.go b/pkg/analyzer/phash_test.go new file mode 100644 index 00000000..463f2d05 --- /dev/null +++ b/pkg/analyzer/phash_test.go @@ -0,0 +1,91 @@ +package analyzer + +import ( + "fmt" + "image" + "image/color" + "math/rand" + "os" + "testing" + + "github.com/azr/phash" + "github.com/disintegration/imaging" +) + +func TestPerceptualHashCompatibility(t *testing.T) { + check := func(t *testing.T, img image.Image) { + t.Helper() + want := phash.DTC(img) + if got := perceptualHash(img); got != want { + t.Fatalf("hash = %016x, reference = %016x", got, want) + } + } + t.Run("nil", func(t *testing.T) { check(t, nil) }) + for _, name := range []string{"catsink.jpg", "frame1.png", "frame2.png", "animated.png"} { + t.Run(name, func(t *testing.T) { + f, err := os.Open("testdata/" + name) + if err != nil { + t.Fatal(err) + } + defer f.Close() + img, _, err := image.Decode(f) + if err != nil { + t.Fatal(err) + } + check(t, img) + // InspectImage first resizes wide images to 128 pixels. + check(t, imaging.Resize(img, 128, 0, imaging.Lanczos)) + }) + } + + // Constant and nearly constant images exercise coefficients near the median, + // where even small floating-point changes could flip a hash bit. + for _, c := range []color.NRGBA{ + {0, 0, 0, 255}, {255, 255, 255, 255}, {127, 127, 127, 255}, + {255, 0, 0, 255}, {0, 255, 0, 255}, {0, 0, 255, 255}, + {200, 80, 35, 0}, {200, 80, 35, 1}, {200, 80, 35, 128}, + } { + t.Run(fmt.Sprintf("solid-%v", c), func(t *testing.T) { + img := image.NewNRGBA(image.Rect(0, 0, 32, 32)) + for y := range 32 { + for x := range 32 { + img.SetNRGBA(x, y, c) + } + } + check(t, img) + img.SetNRGBA(13, 19, color.NRGBA{c.R ^ 1, c.G, c.B, c.A}) + check(t, img) + }) + } + + for seed := range int64(32) { + t.Run(fmt.Sprintf("random-%d", seed), func(t *testing.T) { + rng := rand.New(rand.NewSource(seed)) + width, height := 1+rng.Intn(160), 1+rng.Intn(160) + bounds := image.Rect(-3, 5, width-3, height+5) + nrgba := image.NewNRGBA(bounds) + rgba := image.NewRGBA(bounds) + gray := image.NewGray(bounds) + nrgba64 := image.NewNRGBA64(bounds) + paletted := image.NewPaletted(bounds, color.Palette{color.Transparent, color.Black, color.White, color.NRGBA{255, 40, 100, 128}}) + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + c := color.NRGBA{uint8(rng.Intn(256)), uint8(rng.Intn(256)), uint8(rng.Intn(256)), uint8(rng.Intn(256))} + nrgba.SetNRGBA(x, y, c) + rgba.Set(x, y, c) + gray.Set(x, y, c) + nrgba64.Set(x, y, c) + paletted.SetColorIndex(x, y, uint8(rng.Intn(4))) + } + } + for _, img := range []image.Image{nrgba, rgba, gray, nrgba64, paletted} { + check(t, img) + } + if width > 2 && height > 2 { + // Subimages keep the parent's stride and have nonzero origins. + check(t, nrgba.SubImage(bounds.Inset(1))) + check(t, rgba.SubImage(bounds.Inset(1))) + } + }) + } +} diff --git a/pkg/archive/archive_zip.go b/pkg/archive/archive_zip.go index 67acf9ee..706eff89 100644 --- a/pkg/archive/archive_zip.go +++ b/pkg/archive/archive_zip.go @@ -241,10 +241,7 @@ func (e *gozipArchiveEntry) Stream(w io.Writer, start int64, end int64) (int64, if start >= size { return 0, nil } - length = end - start + 1 - if length > size-start { - length = size - start - } + length = min(end-start+1, size-start) } if n, err, handled := e.streamRawFromFile(w, start, length); handled { return n, err diff --git a/pkg/content/element/element.go b/pkg/content/element/element.go index f1cbcefb..ca6d0cd9 100644 --- a/pkg/content/element/element.go +++ b/pkg/content/element/element.go @@ -22,8 +22,8 @@ type Element interface { Locator() manifest.Locator // Locator targeting this element in the Publication. } -func ElementToMap(e Element) map[string]interface{} { - res := make(map[string]interface{}) +func ElementToMap(e Element) map[string]any { + res := make(map[string]any) res["locator"] = e.Locator() if l := e.Language(); l != "" { res["language"] = l @@ -223,9 +223,9 @@ func (e TextElement) Role() TextRole { func (e TextElement) MarshalJSON() ([]byte, error) { res := ElementToMap(e) res["role"] = e.role.Role() - textElements := make([]interface{}, len(e.segments)) + textElements := make([]any, len(e.segments)) for i, s := range e.segments { - te := map[string]interface{}{ + te := map[string]any{ "locator": s.Locator, "text": s.Text, } diff --git a/pkg/content/iterator/html_converter.go b/pkg/content/iterator/html_converter.go index ef414b33..6db947f7 100644 --- a/pkg/content/iterator/html_converter.go +++ b/pkg/content/iterator/html_converter.go @@ -343,7 +343,7 @@ func (c *HTMLConverter) Head(n *html.Node, depth int) { Title: c.baseLocator.Title, Text: c.baseLocator.Text, Locations: manifest.Locations{ - OtherLocations: map[string]interface{}{ + OtherLocations: map[string]any{ "cssSelector": cssSelector, }, }, @@ -519,7 +519,7 @@ func (c *HTMLConverter) flushText() { MediaType: c.baseLocator.MediaType, Title: c.baseLocator.Title, Locations: manifest.Locations{ - OtherLocations: map[string]interface{}{}, + OtherLocations: map[string]any{}, }, Text: trimText(c.elementRawTextAcc, before), }, @@ -571,7 +571,7 @@ func (c *HTMLConverter) flushSegment() { Title: c.baseLocator.Title, Locations: manifest.Locations{ // TODO fix: needs to use baseLocator locations too! - OtherLocations: map[string]interface{}{}, + OtherLocations: map[string]any{}, }, Text: trimText(c.rawTextAcc, before), }, diff --git a/pkg/fetcher/fetcher_archive.go b/pkg/fetcher/fetcher_archive.go index 169a6a35..76bcd556 100644 --- a/pkg/fetcher/fetcher_archive.go +++ b/pkg/fetcher/fetcher_archive.go @@ -5,6 +5,7 @@ import ( "errors" "io" "path" + "sync" "github.com/readium/go-toolkit/pkg/archive" "github.com/readium/go-toolkit/pkg/manifest" @@ -58,19 +59,16 @@ func (f *ArchiveFetcher) Get(ctx context.Context, link manifest.Link) Resource { // Compute archive properties cl := entry.CompressedLength() + isCompressed := cl > 0 if cl == 0 { cl = entry.Length() } er := &entryResource{ - link: link, - entry: entry, - properties: manifest.Properties{ - "https://readium.org/webpub-manifest/properties#archive": map[string]interface{}{ - "entryLength": cl, - "isEntryCompressed": entry.CompressedLength() > 0, - }, - }, + link: link, + entry: entry, + entryLength: cl, + isCompressed: isCompressed, } return er @@ -134,9 +132,15 @@ func NewArchiveFetcherFromURLWithFactoryAndContext(ctx context.Context, url url. // Resource from archive entry type entryResource struct { - link manifest.Link - entry archive.Entry - properties manifest.Properties + link manifest.Link + entry archive.Entry + entryLength uint64 + isCompressed bool + + // Most metadata reads only need the entry bytes or length. Allocate the + // extension maps only when requested, keeping one mutable map per resource. + propertiesOnce sync.Once + properties manifest.Properties } // File implements Resource @@ -156,6 +160,14 @@ func (r *entryResource) Link() manifest.Link { // Properties implements Resource func (r *entryResource) Properties() manifest.Properties { + r.propertiesOnce.Do(func() { + r.properties = manifest.Properties{ + "https://readium.org/webpub-manifest/properties#archive": map[string]any{ + "entryLength": r.entryLength, + "isEntryCompressed": r.isCompressed, + }, + } + }) return r.properties } diff --git a/pkg/fetcher/fetcher_archive_properties_test.go b/pkg/fetcher/fetcher_archive_properties_test.go new file mode 100644 index 00000000..00586b0b --- /dev/null +++ b/pkg/fetcher/fetcher_archive_properties_test.go @@ -0,0 +1,125 @@ +package fetcher + +import ( + "sync" + "testing" + + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Properties are mutable extension metadata. Calls on one resource must keep +// returning the same maps, including when their first access is concurrent. +func TestArchiveResourcePropertiesConcurrent(t *testing.T) { + f, err := NewArchiveFetcherFromPath(t.Context(), "./testdata/epub.epub") + require.NoError(t, err) + t.Cleanup(f.Close) + r := f.Get(t.Context(), manifest.Link{ + Href: manifest.MustNewHREFFromString("EPUB/css/epub.css", false), + }) + t.Cleanup(r.Close) + + const readers = 32 + properties := make([]manifest.Properties, readers) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range properties { + wg.Go(func() { + <-start + properties[i] = r.Properties() + }) + } + close(start) + wg.Wait() + + const archiveKey = "https://readium.org/webpub-manifest/properties#archive" + for _, p := range properties { + require.Equal(t, map[string]any{ + "entryLength": uint64(595), + "isEntryCompressed": true, + }, p[archiveKey]) + } + + // Mutate only after the concurrent readers finish: callers still need to + // synchronize their own modifications to the returned maps. + properties[0]["custom"] = "extension" + properties[0][archiveKey].(map[string]any)["custom"] = "nested extension" + for _, p := range properties { + assert.Equal(t, "extension", p["custom"]) + assert.Equal(t, "nested extension", p[archiveKey].(map[string]any)["custom"]) + } + assert.Equal(t, "extension", r.Properties()["custom"]) +} + +func TestArchiveResourcePropertiesIndependent(t *testing.T) { + f, err := NewArchiveFetcherFromPath(t.Context(), "./testdata/epub.epub") + require.NoError(t, err) + t.Cleanup(f.Close) + link := manifest.Link{Href: manifest.MustNewHREFFromString("mimetype", false)} + first := f.Get(t.Context(), link) + second := f.Get(t.Context(), link) + t.Cleanup(first.Close) + t.Cleanup(second.Close) + + const archiveKey = "https://readium.org/webpub-manifest/properties#archive" + first.Properties()["custom"] = "extension" + first.Properties()[archiveKey].(map[string]any)["entryLength"] = uint64(1) + assert.Equal(t, manifest.Properties{ + archiveKey: map[string]any{ + "entryLength": uint64(20), + "isEntryCompressed": false, + }, + }, second.Properties()) +} + +// The archive stays open and its entry cache is warmed before timing. Each +// iteration gets a new resource, as happens when parsing metadata or generating +// positions, and either reads its length or asks for archive properties. +func BenchmarkArchiveFetcherGet(b *testing.B) { + for _, entry := range []struct { + name string + path string + }{ + {"Stored", "mimetype"}, + {"Deflated", "EPUB/css/epub.css"}, + } { + b.Run(entry.name, func(b *testing.B) { + f, err := NewArchiveFetcherFromPath(b.Context(), "./testdata/epub.epub") + if err != nil { + b.Fatal(err) + } + b.Cleanup(f.Close) + link := manifest.Link{Href: manifest.MustNewHREFFromString(entry.path, false)} + warm := f.Get(b.Context(), link) + length, rerr := warm.Length(b.Context()) + if rerr != nil { + b.Fatal(rerr) + } + warm.Close() + + b.Run("Length", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + r := f.Get(b.Context(), link) + n, rerr := r.Length(b.Context()) + r.Close() + if rerr != nil || n != length { + b.Fatalf("Length = %d, %v; want %d, nil", n, rerr, length) + } + } + }) + b.Run("Properties", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + r := f.Get(b.Context(), link) + p := r.Properties() + r.Close() + if len(p) != 1 { + b.Fatalf("expected archive properties, got %v", p) + } + } + }) + }) + } +} diff --git a/pkg/fetcher/fetcher_archive_test.go b/pkg/fetcher/fetcher_archive_test.go index e7dac3c8..8cc24947 100644 --- a/pkg/fetcher/fetcher_archive_test.go +++ b/pkg/fetcher/fetcher_archive_test.go @@ -27,7 +27,7 @@ func TestArchiveFetcherLinks(t *testing.T) { MediaType: typ, } p := manifest.Properties{ - "https://readium.org/webpub-manifest/properties#archive": map[string]interface{}{ + "https://readium.org/webpub-manifest/properties#archive": map[string]any{ "entryLength": entryLength, "isEntryCompressed": isCompressed, }, @@ -142,7 +142,7 @@ func TestArchiveFetcherAddsProperties(t *testing.T) { withArchiveFetcher(t, func(a *ArchiveFetcher) { resource := a.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString("EPUB/css/epub.css", false)}) assert.Equal(t, manifest.Properties{ - "https://readium.org/webpub-manifest/properties#archive": map[string]interface{}{ + "https://readium.org/webpub-manifest/properties#archive": map[string]any{ "entryLength": uint64(595), "isEntryCompressed": true, }, diff --git a/pkg/fetcher/fetcher_file.go b/pkg/fetcher/fetcher_file.go index 72c8f90f..4dd60703 100644 --- a/pkg/fetcher/fetcher_file.go +++ b/pkg/fetcher/fetcher_file.go @@ -89,8 +89,8 @@ func (f *FileFetcher) Get(ctx context.Context, link manifest.Link) Resource { linkHref = link.Href.String() } for itemHref, itemFile := range f.paths { - if strings.HasPrefix(linkHref, itemHref) { - resourceFile := filepath.Join(itemFile, strings.TrimPrefix(linkHref, itemHref)) + if after, ok := strings.CutPrefix(linkHref, itemHref); ok { + resourceFile := filepath.Join(itemFile, after) // Make sure that the requested resource is [path] or one of its descendant. rapath, err := filepath.Abs(filepath.ToSlash(resourceFile)) if err != nil { diff --git a/pkg/fetcher/fetcher_gcs.go b/pkg/fetcher/fetcher_gcs.go index d769fd4f..60b704e2 100644 --- a/pkg/fetcher/fetcher_gcs.go +++ b/pkg/fetcher/fetcher_gcs.go @@ -137,8 +137,8 @@ func (f *GCSFetcher) Get(ctx context.Context, link manifest.Link) Resource { } else { linkHref = link.Href.String() } - if strings.HasPrefix(linkHref, f.href) { - resourceFile := path.Join(f.handle.ObjectName(), strings.TrimPrefix(linkHref, f.href)) + if after, ok := strings.CutPrefix(linkHref, f.href); ok { + resourceFile := path.Join(f.handle.ObjectName(), after) // Keep the resource within the fetcher's object-name prefix: a `..` in the HREF // must not let an incoming request reach objects elsewhere in the bucket. if keyWithinRoot(f.handle.ObjectName(), resourceFile) { diff --git a/pkg/fetcher/fetcher_http.go b/pkg/fetcher/fetcher_http.go index b1ad7325..5c65cb0d 100644 --- a/pkg/fetcher/fetcher_http.go +++ b/pkg/fetcher/fetcher_http.go @@ -70,8 +70,8 @@ func (f *HTTPFetcher) Links(ctx context.Context) (manifest.LinkList, error) { // Get implements Fetcher func (f *HTTPFetcher) Get(ctx context.Context, link manifest.Link) Resource { linkHref := link.Href.String() - if strings.HasPrefix(linkHref, f.href) { - rurl, err := url.RelativeURLFromString(strings.TrimPrefix(linkHref, f.href)) + if after, ok := strings.CutPrefix(linkHref, f.href); ok { + rurl, err := url.RelativeURLFromString(after) if err == nil { resolved := f.url.Resolve(rurl).(url.AbsoluteURL) // Keep the resource within the base URL: a `..` or root-absolute HREF must diff --git a/pkg/fetcher/fetcher_s3.go b/pkg/fetcher/fetcher_s3.go index d4c129a3..041bb24f 100644 --- a/pkg/fetcher/fetcher_s3.go +++ b/pkg/fetcher/fetcher_s3.go @@ -123,8 +123,8 @@ func (f *S3Fetcher) Get(ctx context.Context, link manifest.Link) Resource { } else { linkHref = link.Href.String() } - if strings.HasPrefix(linkHref, f.href) { - resourceFile := path.Join(f.key, strings.TrimPrefix(linkHref, f.href)) + if after, ok := strings.CutPrefix(linkHref, f.href); ok { + resourceFile := path.Join(f.key, after) // Keep the resource within the fetcher's key prefix: a `..` in the HREF must // not let an incoming request reach objects elsewhere in the bucket. if keyWithinRoot(f.key, resourceFile) { diff --git a/pkg/fetcher/resource.go b/pkg/fetcher/resource.go index d75f948b..4ed5de42 100644 --- a/pkg/fetcher/resource.go +++ b/pkg/fetcher/resource.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "os" + "unicode/utf8" "github.com/antchfx/xmlquery" "github.com/readium/go-toolkit/pkg/archive" @@ -60,31 +61,36 @@ type Resource interface { } func ReadResourceAsString(ctx context.Context, r Resource) (string, *ResourceError) { - bytes, ex := r.Read(ctx, 0, 0) + bin, ex := r.Read(ctx, 0, 0) if ex != nil { return "", ex } var cs encoding.Encoding - if r.Link().MediaType != nil { - cs = r.Link().MediaType.Charset() + if mediaType := r.Link().MediaType; mediaType != nil { + cs = mediaType.Charset() } if cs == nil { cs = unicode.UTF8 } - utf8bytes, err := cs.NewDecoder().Bytes(bytes) + // Valid UTF-8 needs no decoding. Keep the decoder for malformed input so + // replacement characters and legacy character encodings remain unchanged. + if cs == unicode.UTF8 && utf8.Valid(bin) { + return string(bin), nil + } + utf8bytes, err := cs.NewDecoder().Bytes(bin) if err != nil { return "", Other(err) } return string(utf8bytes), nil } -func ReadResourceAsJSON(ctx context.Context, r Resource) (map[string]interface{}, *ResourceError) { +func ReadResourceAsJSON(ctx context.Context, r Resource) (map[string]any, *ResourceError) { str, ex := ReadResourceAsString(ctx, r) if ex != nil { return nil, ex } - var object map[string]interface{} + var object map[string]any err := json.Unmarshal([]byte(str), &object) if err != nil { return nil, Other(err) diff --git a/pkg/fetcher/resource_string_test.go b/pkg/fetcher/resource_string_test.go new file mode 100644 index 00000000..a13616c2 --- /dev/null +++ b/pkg/fetcher/resource_string_test.go @@ -0,0 +1,171 @@ +package fetcher + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/readium/go-toolkit/pkg/mediatype" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/unicode" +) + +func TestReadResourceAsStringUTF8MatchesDecoder(t *testing.T) { + inputs := []struct { + name string + data []byte + }{ + {"empty", []byte{}}, + {"ASCII", []byte("

Chapter 1\nA new paragraph.

")}, + {"multilingual", []byte("Français 日本語 العربية हिन्दी 🦉")}, + {"BOM", []byte("\ufeff

日本語

")}, + {"embeddedBOM", []byte("before\ufeffafter")}, + {"NUL", []byte("before\x00after")}, + {"invalidByte", []byte("before\xffafter")}, + {"overlong", []byte{0xc0, 0xaf}}, + {"surrogate", []byte{0xed, 0xa0, 0x80}}, + {"truncated", []byte{0xe2, 0x82}}, + {"aboveUnicodeLimit", []byte{0xf4, 0x90, 0x80, 0x80}}, + } + for _, charset := range []struct { + name string + mediaType *mediatype.MediaType + }{ + {"noMediaType", nil}, + {"default", &mediatype.HTML}, + {"UTF8", mediatype.MaybeNewOfString("text/plain;charset=utf-8")}, + {"unknownCharset", mediatype.MaybeNewOfString("text/plain;charset=unknown-test-charset")}, + } { + for _, input := range inputs { + t.Run(charset.name+"/"+input.name, func(t *testing.T) { + resource := &BytesResource{link: manifest.Link{MediaType: charset.mediaType}, _bytes: input.data} + want, wantErr := readResourceAsStringDecoder(t.Context(), resource) + got, gotErr := ReadResourceAsString(t.Context(), resource) + require.Nil(t, wantErr) + require.Nil(t, gotErr) + assert.Equal(t, want, got) + if strings.Contains(input.name, "BOM") { + assert.Contains(t, got, "\ufeff") + } + }) + } + } +} + +func TestReadResourceAsStringLegacyEncodings(t *testing.T) { + for _, test := range []struct { + charset string + data []byte + want string + }{ + {"windows-1252", []byte("caf\xe9 \x80"), "café €"}, + {"iso-8859-1", []byte("caf\xe9 \x80"), "café €"}, + // These bytes are valid UTF-8, but the declared charset still controls + // their interpretation, including embedded NULs in UTF-16 input. + {"windows-1252", []byte("é"), "é"}, + {"utf-16le", []byte("A\x00B\x00"), "AB"}, + {"utf-16be", []byte("\x00A\x00B"), "AB"}, + {"shift_jis", []byte{0x93, 0xfa, 0x96, 0x7b}, "日本"}, + } { + t.Run(test.charset+"/"+test.want, func(t *testing.T) { + mediaType, err := mediatype.NewOfString("text/plain;charset=" + test.charset) + require.NoError(t, err) + resource := &BytesResource{link: manifest.Link{MediaType: &mediaType}, _bytes: test.data} + want, wantErr := readResourceAsStringDecoder(t.Context(), resource) + got, gotErr := ReadResourceAsString(t.Context(), resource) + require.Nil(t, wantErr) + require.Nil(t, gotErr) + assert.Equal(t, test.want, want) + assert.Equal(t, want, got) + }) + } +} + +func TestReadResourceAsStringFailure(t *testing.T) { + wantErr := NotFound(errors.New("missing chapter")) + got, gotErr := ReadResourceAsString(t.Context(), NewFailureResource(manifest.Link{}, wantErr)) + assert.Empty(t, got) + assert.Same(t, wantErr, gotErr) +} + +func TestReadResourceAsStringOwnsReturnedBytes(t *testing.T) { + data := []byte("Chapter 1 日本語") + resource := &BytesResource{_bytes: data} + got, err := ReadResourceAsString(t.Context(), resource) + require.Nil(t, err) + clear(data) + assert.Equal(t, "Chapter 1 日本語", got) +} + +// readResourceAsStringDecoder retains the original decoding path for output +// equivalence checks and side-by-side benchmarks. +func readResourceAsStringDecoder(ctx context.Context, r Resource) (string, *ResourceError) { + bin, ex := r.Read(ctx, 0, 0) + if ex != nil { + return "", ex + } + var cs encoding.Encoding + if r.Link().MediaType != nil { + cs = r.Link().MediaType.Charset() + } + if cs == nil { + cs = unicode.UTF8 + } + utf8bytes, err := cs.NewDecoder().Bytes(bin) + if err != nil { + return "", Other(err) + } + return string(utf8bytes), nil +} + +func BenchmarkReadResourceAsString(b *testing.B) { + for _, input := range []struct { + name, text string + mediaType *mediatype.MediaType + }{ + {"ASCII", "

Chapter one: a paragraph of publication content.

\n", &mediatype.HTML}, + {"multilingual", "

Français 日本語 العربية हिन्दी 🦉

\n", mediatype.MaybeNewOfString("text/html;charset=utf-8")}, + {"invalidUTF8", "

Malformed byte \xff in publication content.

\n", &mediatype.HTML}, + {"legacy", "

Un caf\xe9 co\xfbte 5 \x80.

\n", mediatype.MaybeNewOfString("text/html;charset=windows-1252")}, + } { + for _, size := range []int{4 << 10, 64 << 10, 512 << 10} { + b.Run(fmt.Sprintf("%s/%dKiB", input.name, size>>10), func(b *testing.B) { + data := bytes.Repeat([]byte(input.text), (size+len(input.text)-1)/len(input.text)) + resource := &BytesResource{link: manifest.Link{MediaType: input.mediaType}, _bytes: data} + want, err := readResourceAsStringDecoder(b.Context(), resource) + if err != nil { + b.Fatal(err) + } + for _, implementation := range []struct { + name string + read func(context.Context, Resource) (string, *ResourceError) + }{ + {"decoder", readResourceAsStringDecoder}, + {"optimized", ReadResourceAsString}, + } { + b.Run(implementation.name, func(b *testing.B) { + got, err := implementation.read(b.Context(), resource) + if err != nil || got != want { + b.Fatalf("decoder output mismatch: %v", err) + } + b.ReportAllocs() + b.SetBytes(int64(len(data))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + got, err := implementation.read(b.Context(), resource) + if err != nil || len(got) != len(want) { + b.Fatalf("read failed: %v", err) + } + } + }) + } + }) + } + } +} diff --git a/pkg/guidednavigation/converter/html.go b/pkg/guidednavigation/converter/html.go index 69e70db0..25d53ec3 100644 --- a/pkg/guidednavigation/converter/html.go +++ b/pkg/guidednavigation/converter/html.go @@ -276,12 +276,20 @@ type navigationObject struct { func (n *navigationObject) convert() guidednavigation.GuidedNavigationObject { result := n.object + childrenReserved := false - for _, child := range n.children { + for i, child := range n.children { res := child.convert() if res.Empty() { continue } + if !childrenReserved { + // Reserve room for the remaining children once, instead of repeatedly + // copying large navigation objects as the slice grows. Wait until a + // child is emitted so an empty result retains its original nilness. + result.Children = slices.Grow(result.Children, len(n.children)-i) + childrenReserved = true + } if res.ChildrenOnly() { // Splice transparent wrappers (e.g. plain
s) into the parent result.Children = append(result.Children, res.Children...) @@ -1154,7 +1162,7 @@ func startsWithBindingPunct(s string) bool { if s == "" { return false } - switch []rune(s)[0] { + switch s[0] { case '.', ',', ';', ':', '!', '?', ')', ']', '}': return true } diff --git a/pkg/guidednavigation/converter/html_tree_test.go b/pkg/guidednavigation/converter/html_tree_test.go new file mode 100644 index 00000000..9ff055c2 --- /dev/null +++ b/pkg/guidednavigation/converter/html_tree_test.go @@ -0,0 +1,45 @@ +package converter + +import ( + "testing" + + "github.com/readium/go-toolkit/pkg/guidednavigation" + "github.com/stretchr/testify/assert" +) + +func TestNavigationObjectConvertRetainsChildrenAndOrder(t *testing.T) { + text := func(id string) guidednavigation.GuidedNavigationObject { + return guidednavigation.GuidedNavigationObject{ + ID: id, + Text: guidednavigation.GuidedNavigationText{Plain: id}, + } + } + existing, direct, first, second := text("existing"), text("direct"), text("first"), text("second") + root := navigationObject{ + object: guidednavigation.GuidedNavigationObject{ + Children: []guidednavigation.GuidedNavigationObject{existing}, + }, + children: []*navigationObject{ + {}, // Empty nodes do not produce children. + {object: direct}, + {children: []*navigationObject{ // Transparent wrappers flatten in place. + {object: first}, + {object: second}, + }}, + }, + } + + result := root.convert() + assert.Equal(t, []guidednavigation.GuidedNavigationObject{existing, direct, first, second}, result.Children) + assert.Equal(t, []guidednavigation.GuidedNavigationObject{existing}, root.object.Children) +} + +func TestNavigationObjectConvertEmptyChildrenNilness(t *testing.T) { + for _, children := range [][]guidednavigation.GuidedNavigationObject{nil, {}} { + root := navigationObject{ + object: guidednavigation.GuidedNavigationObject{Children: children}, + children: []*navigationObject{{}, {children: []*navigationObject{{}}}}, + } + assert.Equal(t, children, root.convert().Children) + } +} diff --git a/pkg/guidednavigation/converter/roles.go b/pkg/guidednavigation/converter/roles.go index bb958a55..e87309ff 100644 --- a/pkg/guidednavigation/converter/roles.go +++ b/pkg/guidednavigation/converter/roles.go @@ -25,8 +25,8 @@ func ExtractNamespaces(el *html.Node) (namespaces map[string]string) { // Only the first xmlns gets set namespaces[""] = at.Val } - } else if strings.HasPrefix(at.Key, "xmlns:") { - namespace := strings.TrimPrefix(at.Key, "xmlns:") + } else if after, ok := strings.CutPrefix(at.Key, "xmlns:"); ok { + namespace := after if _, ok := namespaces[namespace]; !ok { // Only the first unique xmlns:prefix gets set namespaces[namespace] = at.Val @@ -237,13 +237,13 @@ func ExtractNodeRoles(el *html.Node) (roles []guidednavigation.GuidedNavigationR presentational := false for _, at := range el.Attr { // Remove namespace prefix if it exists - frags := strings.SplitN(at.Key, ":", 2) - key := frags[len(frags)-1] + prefix, key, hasPrefix := strings.Cut(at.Key, ":") - if len(frags) == 1 { + if !hasPrefix { + key = prefix // ARIA role if key == "role" { - for _, val := range strings.Fields(at.Val) { + for val := range strings.FieldsSeq(at.Val) { if val == "presentation" || val == "none" { presentational = true } @@ -272,14 +272,14 @@ func ExtractNodeRoles(el *html.Node) (roles []guidednavigation.GuidedNavigationR // Save namespaces so they're only extracted once per element namespaces = ExtractNamespaces(el) } - if namespace, ok := namespaces[frags[0]]; ok { + if namespace, ok := namespaces[prefix]; ok { // Set the namespace if we found it at.Namespace = namespace } } if at.Namespace == "http://www.idpf.org/2007/ops" && key == "type" { - for _, val := range strings.Fields(at.Val) { + for val := range strings.FieldsSeq(at.Val) { if role, ok := epubTypeRoles[val]; ok { attrRoles = append(attrRoles, role) } diff --git a/pkg/guidednavigation/converter/xhtml.go b/pkg/guidednavigation/converter/xhtml.go index 0a5edb40..3b39fea8 100644 --- a/pkg/guidednavigation/converter/xhtml.go +++ b/pkg/guidednavigation/converter/xhtml.go @@ -80,8 +80,11 @@ func ParseXHTML(r io.Reader) (*html.Node, error) { DataAtom: atom.Lookup([]byte(local)), Namespace: elementNamespace(t.Name.Space), } - for _, a := range t.Attr { - n.Attr = append(n.Attr, attrFromXML(a, scope)) + if len(t.Attr) > 0 { + n.Attr = make([]html.Attribute, len(t.Attr)) + for i, a := range t.Attr { + n.Attr[i] = attrFromXML(a, scope) + } } cur.AppendChild(n) cur = n diff --git a/pkg/guidednavigation/fragments.go b/pkg/guidednavigation/fragments.go index 32731db3..f1dd60f1 100644 --- a/pkg/guidednavigation/fragments.go +++ b/pkg/guidednavigation/fragments.go @@ -3,6 +3,7 @@ package guidednavigation import ( "math" nurl "net/url" + "slices" "strconv" "strings" "time" @@ -286,11 +287,11 @@ func fragmentDimensions(ref url.URL) []fragmentDimension { // https://www.w3.org/TR/media-frags/#error-uri-general func lastValidDimension[T any](ref url.URL, name string, parse func(string) *T) *T { dims := fragmentDimensions(ref) - for i := len(dims) - 1; i >= 0; i-- { - if dims[i].name != name { + for _, dim := range slices.Backward(dims) { + if dim.name != name { continue } - if res := parse(dims[i].value); res != nil { + if res := parse(dim.value); res != nil { return res } } diff --git a/pkg/guidednavigation/guided_navigation.go b/pkg/guidednavigation/guided_navigation.go index b42c0bf0..2ec5aa67 100644 --- a/pkg/guidednavigation/guided_navigation.go +++ b/pkg/guidednavigation/guided_navigation.go @@ -17,15 +17,15 @@ type GuidedNavigationDocument struct { // Readium Guided Navigation Object // https://readium.org/guided-navigation/schema/object.schema.json type GuidedNavigationObject struct { - ID string `json:"id,omitempty"` // Identifier for the object, referenced from the SSML representation of a sibling text (e.g. ). - AudioRef url.URL `json:"audioref,omitempty"` // References an audio resource or a fragment of it. - ImgRef url.URL `json:"imgref,omitempty"` // References an image or a fragment of it. - TextRef url.URL `json:"textref,omitempty"` // References a textual resource or a fragment of it. - VideoRef url.URL `json:"videoref,omitempty"` // References a video resource or a fragment of it. - Text GuidedNavigationText `json:"text,omitempty"` // Textual equivalent of the resources or fragment of the resources referenced by the current Guided Navigation Object. - Role []GuidedNavigationRole `json:"role,omitempty"` // Convey the structural semantics of a publication - Children []GuidedNavigationObject `json:"children,omitempty"` // Items that are children of the containing Guided Navigation Object. - Description GuidedNavigationDescription `json:"description,omitempty"` // Text, audio or image description for the current Guided Navigation Object. + ID string `json:"id,omitempty"` // Identifier for the object, referenced from the SSML representation of a sibling text (e.g. ). + AudioRef url.URL `json:"audioref,omitempty"` // References an audio resource or a fragment of it. + ImgRef url.URL `json:"imgref,omitempty"` // References an image or a fragment of it. + TextRef url.URL `json:"textref,omitempty"` // References a textual resource or a fragment of it. + VideoRef url.URL `json:"videoref,omitempty"` // References a video resource or a fragment of it. + Text GuidedNavigationText `json:"text"` // Textual equivalent of the resources or fragment of the resources referenced by the current Guided Navigation Object. + Role []GuidedNavigationRole `json:"role,omitempty"` // Convey the structural semantics of a publication + Children []GuidedNavigationObject `json:"children,omitempty"` // Items that are children of the containing Guided Navigation Object. + Description GuidedNavigationDescription `json:"description"` // Text, audio or image description for the current Guided Navigation Object. } func (o GuidedNavigationObject) Empty() bool { @@ -47,7 +47,7 @@ func (o GuidedNavigationObject) TextOnly() bool { } func (o GuidedNavigationObject) MarshalJSON() ([]byte, error) { - res := make(map[string]interface{}) + res := make(map[string]any) if o.Empty() { return json.Marshal(res) } @@ -120,7 +120,7 @@ func (t *GuidedNavigationText) UnmarshalJSON(data []byte) error { } func (t GuidedNavigationText) MarshalJSON() ([]byte, error) { - res := make(map[string]interface{}) + res := make(map[string]any) if t.SSML == "" && t.Language == "" { return json.Marshal(t.Plain) @@ -146,7 +146,7 @@ type GuidedNavigationDescription struct { ImgRef url.URL `json:"imgref,omitempty"` // References an image or a fragment of it. TextRef url.URL `json:"textref,omitempty"` // References a textual resource or a fragment of it. VideoRef url.URL `json:"videoref,omitempty"` // References a video resource or a fragment of it. - Text GuidedNavigationText `json:"text,omitempty"` // Textual description. + Text GuidedNavigationText `json:"text"` // Textual description. } // NewTextDescription creates a description carrying only plain text. @@ -213,7 +213,7 @@ func (d GuidedNavigationDescription) MarshalJSON() ([]byte, error) { return json.Marshal(d.Text.Plain) } - res := make(map[string]interface{}) + res := make(map[string]any) if d.AudioRef != nil { if s := d.AudioRef.String(); s != "" { res["audioref"] = s diff --git a/pkg/internal/extensions/generics.go b/pkg/internal/extensions/generics.go index 0ce61b8f..dc0e4caa 100644 --- a/pkg/internal/extensions/generics.go +++ b/pkg/internal/extensions/generics.go @@ -1,5 +1,7 @@ package extensions +import "slices" + import "encoding/json" func Pointer[T any](val T) *T { @@ -7,12 +9,7 @@ func Pointer[T any](val T) *T { } func Contains[T comparable](slice []T, element T) bool { - for _, v := range slice { - if v == element { - return true - } - } - return false + return slices.Contains(slice, element) } func Equal[T comparable](a, b []T) bool { @@ -35,10 +32,8 @@ func AppendIfMissing[T comparable](slice []T, elems ...T) []T { } func AppendElementIfMissing[T comparable](slice []T, elem T) []T { - for _, v := range slice { - if v == elem { - return slice - } + if slices.Contains(slice, elem) { + return slice } return append(slice, elem) } diff --git a/pkg/internal/obfuscation/xor.go b/pkg/internal/obfuscation/xor.go new file mode 100644 index 00000000..c8d9a67e --- /dev/null +++ b/pkg/internal/obfuscation/xor.go @@ -0,0 +1,32 @@ +// Package obfuscation implements the repeating-key XOR used by EPUB fonts. +package obfuscation + +import "crypto/subtle" + +// XOR transforms data in place using a repeating key, starting at the given +// non-negative offset in the original resource. An empty key leaves data intact. +func XOR(data, key []byte, offset int64) { + if len(data) == 0 || len(key) == 0 { + return + } + + // Both EPUB font algorithms fit in this stack buffer (1024 or 1040 bytes). + // Expand the key with bulk copies so the XOR can use the standard library's + // optimized implementation without a modulo operation for every byte. + var expanded [1040]byte + phase := int(offset % int64(len(key))) + for len(data) > 0 { + n := min(len(data), len(expanded)) + mask := expanded[:n] + filled := copy(mask, key[phase:]) + filled += copy(mask[filled:], key[:phase]) + for filled < n { + filled += copy(mask[filled:], mask[:filled]) + } + subtle.XORBytes(data[:n], data[:n], mask) + data = data[n:] + if len(data) > 0 { + phase = (phase + n) % len(key) + } + } +} diff --git a/pkg/internal/obfuscation/xor_test.go b/pkg/internal/obfuscation/xor_test.go new file mode 100644 index 00000000..169794db --- /dev/null +++ b/pkg/internal/obfuscation/xor_test.go @@ -0,0 +1,104 @@ +package obfuscation + +import ( + "bytes" + "fmt" + "testing" +) + +func TestXOR(t *testing.T) { + for _, keyLen := range []int{0, 1, 16, 20, 1057} { + key := make([]byte, keyLen) + for i := range key { + key[i] = byte(i*13 + 7) + } + for _, offset := range []int64{0, 1, 13, 15, 16, 19, 20, 1023, 1040} { + for _, size := range []int{0, 1, 15, 16, 17, 63, 64, 65, 1024, 1040, 1041, 4096} { + t.Run(fmt.Sprintf("key%d/offset%d/size%d", keyLen, offset, size), func(t *testing.T) { + original := make([]byte, size) + for i := range original { + original[i] = byte(i*3 + 1) + } + want := bytes.Clone(original) + if keyLen > 0 { + for i := range want { + want[i] ^= key[(offset+int64(i))%int64(keyLen)] + } + } + got := bytes.Clone(original) + XOR(got, key, offset) + if !bytes.Equal(got, want) { + t.Fatal("result differs from repeating-key XOR") + } + XOR(got, key, offset) + if !bytes.Equal(got, original) { + t.Fatal("second XOR did not restore the original data") + } + }) + } + } + } +} + +// xorModulo preserves the original byte-at-a-time font transform as a benchmark +// baseline. Both implementations receive data already limited to the font prefix. +func xorModulo(data, key []byte, offset int64) { + if len(key) == 0 { + return + } + keyLen := int64(len(key)) + for i := int64(0); i < int64(len(data)); i++ { + data[i] ^= key[(offset+i)%keyLen] + } +} + +// BenchmarkXOR includes key expansion in the optimized measurement, but excludes +// fixture setup, key derivation, and I/O from both implementations. +func BenchmarkXOR(b *testing.B) { + for _, keyLen := range []int{16, 20} { + size := 1024 + if keyLen == 20 { + size = 1040 + } + key := make([]byte, keyLen) + for i := range key { + key[i] = byte(i*13 + 7) + } + for _, span := range []struct { + name string + offset int64 + size int + }{ + {"Full", 0, size}, + {"Unaligned", 13, size - 13}, + {"Short", 13, 16}, + {"Tail", int64(size - 1), 1}, + } { + for _, impl := range []struct { + name string + xor func([]byte, []byte, int64) + }{ + {"Modulo", xorModulo}, + {"XORBytes", XOR}, + } { + b.Run(fmt.Sprintf("key%d/%s/%s", keyLen, span.name, impl.name), func(b *testing.B) { + data := bytes.Repeat([]byte{0x5a}, span.size) + b.SetBytes(int64(len(data))) + b.ReportAllocs() + for b.Loop() { + impl.xor(data, key, span.offset) + } + // Each iteration toggles the same input. Check the final + // bytes after B.Loop has stopped the timer. + want := bytes.Repeat([]byte{0x5a}, span.size) + if b.N%2 != 0 { + xorModulo(want, key, span.offset) + } + if !bytes.Equal(data, want) { + b.Fatal("incorrect benchmark result") + } + }) + } + } + } +} diff --git a/pkg/internal/util/map.go b/pkg/internal/util/map.go index 6db3b7ae..22059d59 100644 --- a/pkg/internal/util/map.go +++ b/pkg/internal/util/map.go @@ -2,5 +2,5 @@ package util type JSONMappable interface { // JSONMap returns a JSON object representation of the receiver. - JSONMap() (map[string]interface{}, error) + JSONMap() (map[string]any, error) } diff --git a/pkg/manifest/a11y.go b/pkg/manifest/a11y.go index b830db03..beeaaef7 100644 --- a/pkg/manifest/a11y.go +++ b/pkg/manifest/a11y.go @@ -76,7 +76,7 @@ func (a *A11y) Merge(other *A11y) { } } -func A11yFromJSON(rawJSON map[string]interface{}) (*A11y, error) { +func A11yFromJSON(rawJSON map[string]any) (*A11y, error) { if rawJSON == nil { return nil, nil } @@ -90,7 +90,7 @@ func A11yFromJSON(rawJSON map[string]interface{}) (*A11y, error) { a.ConformsTo = A11yProfilesFromStrings(conformsTo) a.ConformsTo.Sort() - if certJSON, ok := rawJSON["certification"].(map[string]interface{}); ok { + if certJSON, ok := rawJSON["certification"].(map[string]any); ok { c := A11yCertification{ CertifiedBy: parseOptString(certJSON["certifiedBy"]), Credential: parseOptString(certJSON["credential"]), @@ -110,7 +110,7 @@ func A11yFromJSON(rawJSON map[string]interface{}) (*A11y, error) { a.AccessModes = A11yAccessModesFromStrings(accessModes) ams := [][]A11yPrimaryAccessMode{} - if amsJSON, ok := rawJSON["accessModeSufficient"].([]interface{}); ok { + if amsJSON, ok := rawJSON["accessModeSufficient"].([]any); ok { for _, l := range amsJSON { strings, err := parseSliceOrString(l, true) if err != nil { @@ -145,13 +145,13 @@ func A11yFromJSON(rawJSON map[string]interface{}) (*A11y, error) { } func (e *A11y) UnmarshalJSON(data []byte) error { - var d interface{} + var d any err := json.Unmarshal(data, &d) if err != nil { return err } - mp, ok := d.(map[string]interface{}) + mp, ok := d.(map[string]any) if !ok { return errors.New("accessibility object not a map with string keys") } diff --git a/pkg/manifest/collection.go b/pkg/manifest/collection.go index 68c6adb7..f6b81632 100644 --- a/pkg/manifest/collection.go +++ b/pkg/manifest/collection.go @@ -15,12 +15,12 @@ type PublicationCollectionMap map[string][]PublicationCollection // PublicationCollection can be used as extension point in the Readium Web Publication Manifest. // https://readium.org/webpub-manifest/schema/subcollection.schema.json type PublicationCollection struct { - Metadata map[string]interface{} `json:"metadata,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` Links LinkList `json:"links,omitempty"` Subcollections PublicationCollectionMap `json:"-"` } -func appendPublicationCollectionToJSON(pc PublicationCollectionMap, obj map[string]interface{}) { +func appendPublicationCollectionToJSON(pc PublicationCollectionMap, obj map[string]any) { for role, collections := range pc { if len(collections) == 0 { continue @@ -38,7 +38,7 @@ func appendPublicationCollectionToJSON(pc PublicationCollectionMap, obj map[stri // TODO log [warnings] ? // // The [links]' href and their children's will be normalized recursively using the provided [normalizeHref] closure. -func PublicationCollectionFromJSON(rawJson interface{}) (*PublicationCollection, error) { +func PublicationCollectionFromJSON(rawJson any) (*PublicationCollection, error) { if rawJson == nil { return nil, nil } @@ -46,13 +46,13 @@ func PublicationCollectionFromJSON(rawJson interface{}) (*PublicationCollection, var err error var links []Link // var metadata map[string]interface{} - metadata := make(map[string]interface{}) + metadata := make(map[string]any) var subcollections PublicationCollectionMap //subcollections := make(PublicationCollectionMap) switch dd := rawJson.(type) { - case map[string]interface{}: - lkz, ok := dd["links"].([]interface{}) + case map[string]any: + lkz, ok := dd["links"].([]any) if ok { links, err = LinksFromJSONArray(lkz) if err != nil { @@ -60,7 +60,7 @@ func PublicationCollectionFromJSON(rawJson interface{}) (*PublicationCollection, } } - mtd, ok := dd["metadata"].(map[string]interface{}) + mtd, ok := dd["metadata"].(map[string]any) if ok { metadata = mtd } @@ -72,7 +72,7 @@ func PublicationCollectionFromJSON(rawJson interface{}) (*PublicationCollection, if err != nil { return nil, errors.Wrap(err, "failed unmarshalling subcollections") } - case []interface{}: + case []any: links, err = LinksFromJSONArray(dd) if err != nil { return nil, errors.Wrap(err, "failed unmarshalling as Link array") @@ -93,7 +93,7 @@ func PublicationCollectionFromJSON(rawJson interface{}) (*PublicationCollection, } func (pc *PublicationCollection) UnmarshalJSON(b []byte) error { - var object map[string]interface{} + var object map[string]any err := json.Unmarshal(b, &object) if err != nil { return err @@ -110,7 +110,7 @@ func (pc PublicationCollection) MarshalJSON() ([]byte, error) { if pc.Metadata == nil && pc.Subcollections == nil { return json.Marshal(pc.Links) } - res := make(map[string]interface{}) + res := make(map[string]any) res["metadata"] = pc.Metadata res["links"] = pc.Links appendPublicationCollectionToJSON(pc.Subcollections, res) @@ -121,7 +121,7 @@ func (pc PublicationCollection) MarshalJSON() ([]byte, error) { // Parses a map of [PublicationCollection] indexed by their roles from its RWPM JSON representation. // // The [Links]' href and their children's will be normalized recursively using the provided [normalizeHref] closure. -func PublicationCollectionsFromJSON(rawJson map[string]interface{}) (PublicationCollectionMap, error) { +func PublicationCollectionsFromJSON(rawJson map[string]any) (PublicationCollectionMap, error) { if rawJson == nil { return nil, nil } @@ -144,7 +144,7 @@ func PublicationCollectionsFromJSON(rawJson map[string]interface{}) (Publication } else { collections[role] = []PublicationCollection{*collection} } - } else if subArr, ok := sub.([]interface{}); ok { + } else if subArr, ok := sub.([]any); ok { // Parses a list of collection objects. var newCollections []PublicationCollection for j, v := range subArr { @@ -174,7 +174,7 @@ func PublicationCollectionsFromJSON(rawJson map[string]interface{}) (Publication } func (pcm *PublicationCollectionMap) UnmarshalJSON(b []byte) error { - var object map[string]interface{} + var object map[string]any err := json.Unmarshal(b, &object) if err != nil { return err @@ -192,7 +192,7 @@ func (pcm PublicationCollectionMap) MarshalJSON() ([]byte, error) { return []byte("null"), nil } - res := make(map[string]interface{}) + res := make(map[string]any) for k, v := range pcm { if len(v) == 0 { diff --git a/pkg/manifest/collection_test.go b/pkg/manifest/collection_test.go index 9588b222..b4132984 100644 --- a/pkg/manifest/collection_test.go +++ b/pkg/manifest/collection_test.go @@ -18,7 +18,7 @@ func TestPubCollectionUnmarshalMinimalJSON(t *testing.T) { assert.Equal(t, PublicationCollection{ Links: []Link{{Href: NewHREF(url.MustURLFromString("/link"))}}, - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, }, pc, "unmarshalled JSON object should be equal to PublicationCollection object") } @@ -56,21 +56,21 @@ func TestPubCollectionUnmarshalFullJSON(t *testing.T) { assert.Equal(t, PublicationCollection{ Links: []Link{{Href: NewHREF(url.MustURLFromString("/link"))}}, - Metadata: map[string]interface{}{ + Metadata: map[string]any{ "metadata1": "value", }, Subcollections: PublicationCollectionMap{ "sub1": {{ - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink"))}}, }}, "sub2": {{ - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink1"))}, {Href: NewHREF(url.MustURLFromString("/sublink2"))}}, }}, "sub3": { - {Metadata: map[string]interface{}{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink3"))}}}, - {Metadata: map[string]interface{}{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink4"))}}}, + {Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink3"))}}}, + {Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink4"))}}}, }, }, }, pc, "unmarshalled JSON object should be equal to PublicationCollection object") @@ -83,7 +83,7 @@ func TestPubCollectionUnmarshalNilJSON(t *testing.T) { } func TestPubCollectionUnmarshalJSONMultipleCollections(t *testing.T) { - var pcsr map[string]interface{} + var pcsr map[string]any require.NoError(t, json.Unmarshal([]byte(`{ "sub1": { "links": [ @@ -112,23 +112,23 @@ func TestPubCollectionUnmarshalJSONMultipleCollections(t *testing.T) { assert.Equal(t, PublicationCollectionMap{ "sub1": {{ - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink"))}}, }}, "sub2": {{ - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink1"))}, {Href: NewHREF(url.MustURLFromString("/sublink2"))}}, }}, "sub3": { - {Metadata: map[string]interface{}{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink3"))}}}, - {Metadata: map[string]interface{}{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink4"))}}}, + {Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink3"))}}}, + {Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink4"))}}}, }, }, pcs, "unmarshalled JSON object should be equal to map of PublicationCollection to role") } func TestPubCollectionMinimalJSON(t *testing.T) { bin, err := json.Marshal(&PublicationCollection{ - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/link"))}}, }) require.NoError(t, err) @@ -140,22 +140,22 @@ func TestPubCollectionMinimalJSON(t *testing.T) { func TestPubCollectionFullJSON(t *testing.T) { bin, err := json.Marshal(&PublicationCollection{ - Metadata: map[string]interface{}{ + Metadata: map[string]any{ "metadata1": "value", }, Links: []Link{{Href: NewHREF(url.MustURLFromString("/link"))}}, Subcollections: PublicationCollectionMap{ "sub1": {{ - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink"))}}, }}, "sub2": {{ - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink1"))}, {Href: NewHREF(url.MustURLFromString("/sublink2"))}}, }}, "sub3": { - {Metadata: map[string]interface{}{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink3"))}}}, - {Metadata: map[string]interface{}{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink4"))}}}, + {Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink3"))}}}, + {Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink4"))}}}, }, }, }) @@ -200,16 +200,16 @@ func TestPubCollectionFullJSON(t *testing.T) { func TestPubCollectionMultipleCollectionsJSON(t *testing.T) { bin, err := json.Marshal(PublicationCollectionMap{ "sub1": {{ - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink"))}}, }}, "sub2": {{ - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink1"))}, {Href: NewHREF(url.MustURLFromString("/sublink2"))}}, }}, "sub3": { - {Metadata: map[string]interface{}{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink3"))}}}, - {Metadata: map[string]interface{}{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink4"))}}}, + {Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink3"))}}}, + {Metadata: map[string]any{}, Links: []Link{{Href: NewHREF(url.MustURLFromString("/sublink4"))}}}, }, }) require.NoError(t, err) diff --git a/pkg/manifest/contributor.go b/pkg/manifest/contributor.go index c5773db6..ef7e9677 100644 --- a/pkg/manifest/contributor.go +++ b/pkg/manifest/contributor.go @@ -34,7 +34,7 @@ func (c Contributor) SortAs() string { // Parses a [Contributor] from its RWPM JSON representation. // A contributor can be parsed from a single string, or a full-fledged object. // The [links]' href and their children's will be normalized recursively using the provided [normalizeHref] closure. -func ContributorFromJSON(rawJson interface{}) (*Contributor, error) { +func ContributorFromJSON(rawJson any) (*Contributor, error) { if rawJson == nil { return nil, nil } @@ -43,7 +43,7 @@ func ContributorFromJSON(rawJson interface{}) (*Contributor, error) { switch dd := rawJson.(type) { case string: // Just a single string Contributor c.LocalizedName = NewLocalizedStringFromString(dd) - case map[string]interface{}: // Actual object Contributor + case map[string]any: // Actual object Contributor // LocalizedName nr, ok := dd["name"] if !ok { @@ -74,7 +74,7 @@ func ContributorFromJSON(rawJson interface{}) (*Contributor, error) { c.Roles = roles // Links - rawLinks, ok := dd["links"].([]interface{}) + rawLinks, ok := dd["links"].([]any) if ok { links, err := LinksFromJSONArray(rawLinks) if err != nil { @@ -108,10 +108,10 @@ func ContributorFromJSON(rawJson interface{}) (*Contributor, error) { return c, nil } -func ContributorFromJSONArray(rawJsonArray interface{}) ([]Contributor, error) { +func ContributorFromJSONArray(rawJsonArray any) ([]Contributor, error) { var contributors []Contributor switch rjx := rawJsonArray.(type) { - case []interface{}: + case []any: contributors = make([]Contributor, 0, len(rjx)) for i, entry := range rjx { rc, err := ContributorFromJSON(entry) @@ -145,7 +145,7 @@ func (c Contributor) MarshalJSON() ([]byte, error) { } func (c *Contributor) UnmarshalJSON(data []byte) error { - var d interface{} + var d any err := json.Unmarshal(data, &d) if err != nil { return err diff --git a/pkg/manifest/encryption.go b/pkg/manifest/encryption.go index 2d8b9977..1cee3999 100644 --- a/pkg/manifest/encryption.go +++ b/pkg/manifest/encryption.go @@ -14,7 +14,7 @@ type Encryption struct { OriginalLength int64 `json:"originalLength,omitempty"` } -func EncryptionFromJSON(rawJson map[string]interface{}) (*Encryption, error) { +func EncryptionFromJSON(rawJson map[string]any) (*Encryption, error) { if rawJson == nil { return nil, nil } @@ -38,13 +38,13 @@ func EncryptionFromJSON(rawJson map[string]interface{}) (*Encryption, error) { } func (e *Encryption) UnmarshalJSON(data []byte) error { - var d interface{} + var d any err := json.Unmarshal(data, &d) if err != nil { return err } - mp, ok := d.(map[string]interface{}) + mp, ok := d.(map[string]any) if !ok { return errors.New("encryption object not a map with string keys") } @@ -57,8 +57,8 @@ func (e *Encryption) UnmarshalJSON(data []byte) error { return nil } -func (m Encryption) ToMap() map[string]interface{} { - mp := make(map[string]interface{}) +func (m Encryption) ToMap() map[string]any { + mp := make(map[string]any) mp["algorithm"] = m.Algorithm if m.Compression != "" { mp["compression"] = m.Compression diff --git a/pkg/manifest/link.go b/pkg/manifest/link.go index fec9d395..6922be0b 100644 --- a/pkg/manifest/link.go +++ b/pkg/manifest/link.go @@ -2,6 +2,7 @@ package manifest import ( "encoding/json" + "slices" "github.com/pkg/errors" "github.com/readium/go-toolkit/pkg/mediatype" @@ -34,7 +35,7 @@ func (l Link) URL(base url.URL, parameters map[string]string) url.URL { } // Creates an [Link] from its RWPM JSON representation. -func LinkFromJSON(rawJson map[string]interface{}) (*Link, error) { +func LinkFromJSON(rawJson map[string]any) (*Link, error) { if rawJson == nil { return nil, nil } @@ -85,7 +86,7 @@ func LinkFromJSON(rawJson map[string]interface{}) (*Link, error) { } // Properties - properties, ok := rawJson["properties"].(map[string]interface{}) + properties, ok := rawJson["properties"].(map[string]any) if ok { link.Properties = properties } @@ -105,7 +106,7 @@ func LinkFromJSON(rawJson map[string]interface{}) (*Link, error) { link.Languages = languages // Alternates - rawAlternates, ok := rawJson["alternate"].([]interface{}) + rawAlternates, ok := rawJson["alternate"].([]any) if ok { alternates, err := LinksFromJSONArray(rawAlternates) if err != nil { @@ -115,7 +116,7 @@ func LinkFromJSON(rawJson map[string]interface{}) (*Link, error) { } // Children - rawChildren, ok := rawJson["children"].([]interface{}) + rawChildren, ok := rawJson["children"].([]any) if ok { children, err := LinksFromJSONArray(rawChildren) if err != nil { @@ -127,10 +128,10 @@ func LinkFromJSON(rawJson map[string]interface{}) (*Link, error) { return link, nil } -func LinksFromJSONArray(rawJsonArray []interface{}) (LinkList, error) { +func LinksFromJSONArray(rawJsonArray []any) (LinkList, error) { links := make(LinkList, 0, len(rawJsonArray)) for i, entry := range rawJsonArray { - entry, ok := entry.(map[string]interface{}) + entry, ok := entry.(map[string]any) if !ok { // TODO: Should this be a "warning", an error, or completely ignored? continue @@ -148,7 +149,7 @@ func LinksFromJSONArray(rawJsonArray []interface{}) (LinkList, error) { } func (l *Link) UnmarshalJSON(b []byte) error { - var object map[string]interface{} + var object map[string]any err := json.Unmarshal(b, &object) if err != nil { return err @@ -162,7 +163,7 @@ func (l *Link) UnmarshalJSON(b []byte) error { } func (l Link) MarshalJSON() ([]byte, error) { - res := make(map[string]interface{}) + res := make(map[string]any) res["href"] = l.Href.String() if l.MediaType != nil { res["type"] = l.MediaType.String() @@ -232,10 +233,8 @@ func (ll LinkList) FirstWithHref(href url.URL) *Link { // Finds the first link with the given relation. func (ll LinkList) FirstWithRel(rel string) *Link { for _, link := range ll { - for _, r := range link.Rels { - if r == rel { - return &link - } + if slices.Contains(link.Rels, rel) { + return &link } } return nil diff --git a/pkg/manifest/localized_string.go b/pkg/manifest/localized_string.go index 75d223a8..374ff0ba 100644 --- a/pkg/manifest/localized_string.go +++ b/pkg/manifest/localized_string.go @@ -102,7 +102,7 @@ func (l LocalizedString) MarshalJSON() ([]byte, error) { return json.Marshal(l.DefaultTranslation()) } -func LocalizedStringFromJSON(d interface{}) (*LocalizedString, error) { +func LocalizedStringFromJSON(d any) (*LocalizedString, error) { if d == nil { return nil, errors.New("LocalizedString is nil") } @@ -110,7 +110,7 @@ func LocalizedStringFromJSON(d interface{}) (*LocalizedString, error) { switch lsx := d.(type) { case string: l.SetDefaultTranslation(lsx) - case map[string]interface{}: + case map[string]any: for k, v := range lsx { val, ok := v.(string) if !ok { @@ -125,7 +125,7 @@ func LocalizedStringFromJSON(d interface{}) (*LocalizedString, error) { } func (l *LocalizedString) UnmarshalJSON(data []byte) error { - var d interface{} + var d any err := json.Unmarshal(data, &d) if err != nil { return err diff --git a/pkg/manifest/locator.go b/pkg/manifest/locator.go index 3aae4db4..ca1af466 100644 --- a/pkg/manifest/locator.go +++ b/pkg/manifest/locator.go @@ -2,6 +2,7 @@ package manifest import ( "encoding/json" + "maps" "github.com/pkg/errors" "github.com/readium/go-toolkit/pkg/mediatype" @@ -11,14 +12,14 @@ import ( // One or more alternative expressions of the location. // https://github.com/readium/architecture/tree/master/models/locators#the-location-object type Locations struct { - Fragments []string `json:"fragments,omitempty"` // Contains one or more fragment in the resource referenced by the [Locator]. - Progression *float64 `json:"progression,omitempty"` // Progression in the resource expressed as a percentage (between 0 and 1). - Position *uint `json:"position,omitempty"` // An index in the publication (>= 1). - TotalProgression *float64 `json:"totalProgression,omitempty"` // Progression in the publication expressed as a percentage (between 0 and 1). - OtherLocations map[string]interface{} // Additional locations for extensions. + Fragments []string `json:"fragments,omitempty"` // Contains one or more fragment in the resource referenced by the [Locator]. + Progression *float64 `json:"progression,omitempty"` // Progression in the resource expressed as a percentage (between 0 and 1). + Position *uint `json:"position,omitempty"` // An index in the publication (>= 1). + TotalProgression *float64 `json:"totalProgression,omitempty"` // Progression in the publication expressed as a percentage (between 0 and 1). + OtherLocations map[string]any // Additional locations for extensions. } -func LocationsFromJSON(rawJson map[string]interface{}) (l Locations, err error) { +func LocationsFromJSON(rawJson map[string]any) (l Locations, err error) { if rawJson == nil { return } @@ -81,7 +82,7 @@ func LocationsFromJSON(rawJson map[string]interface{}) (l Locations, err error) } func (l *Locations) UnmarshalJSON(b []byte) error { - var object map[string]interface{} + var object map[string]any err := json.Unmarshal(b, &object) if err != nil { return err @@ -95,11 +96,9 @@ func (l *Locations) UnmarshalJSON(b []byte) error { } func (l Locations) MarshalJSON() ([]byte, error) { - j := make(map[string]interface{}) + j := make(map[string]any) if l.OtherLocations != nil { - for k, v := range l.OtherLocations { - j[k] = v - } + maps.Copy(j, l.OtherLocations) } if len(l.Fragments) > 0 { @@ -140,7 +139,7 @@ type Text struct { After string `json:"after,omitempty"` // The text after the locator. } -func TextFromJSON(rawJson map[string]interface{}) (t Text) { +func TextFromJSON(rawJson map[string]any) (t Text) { if rawJson == nil { return } @@ -191,11 +190,11 @@ type Locator struct { Href url.URL `json:"href"` MediaType mediatype.MediaType `json:"type"` Title string `json:"title,omitempty"` - Locations Locations `json:"locations,omitempty"` - Text Text `json:"text,omitempty"` + Locations Locations `json:"locations"` + Text Text `json:"text"` } -func LocatorFromJSON(rawJson map[string]interface{}) (Locator, error) { +func LocatorFromJSON(rawJson map[string]any) (Locator, error) { if rawJson == nil { return Locator{}, nil } @@ -221,7 +220,7 @@ func LocatorFromJSON(rawJson map[string]interface{}) (Locator, error) { return Locator{}, errors.Wrap(err, "failed unmarshalling 'type' as valid mimetype") } - if rawLocations, ok := rawJson["locations"].(map[string]interface{}); ok { + if rawLocations, ok := rawJson["locations"].(map[string]any); ok { locations, err := LocationsFromJSON(rawLocations) if err != nil { return Locator{}, err @@ -229,7 +228,7 @@ func LocatorFromJSON(rawJson map[string]interface{}) (Locator, error) { locator.Locations = locations } - if rawText, ok := rawJson["text"].(map[string]interface{}); ok { + if rawText, ok := rawJson["text"].(map[string]any); ok { locator.Text = TextFromJSON(rawText) } @@ -237,7 +236,7 @@ func LocatorFromJSON(rawJson map[string]interface{}) (Locator, error) { } func (l *Locator) UnmarshalJSON(b []byte) error { - var object map[string]interface{} + var object map[string]any err := json.Unmarshal(b, &object) if err != nil { return err @@ -251,7 +250,7 @@ func (l *Locator) UnmarshalJSON(b []byte) error { } func (l Locator) MarshalJSON() ([]byte, error) { - j := make(map[string]interface{}) + j := make(map[string]any) if l.Href == nil { return nil, errors.New("href is required in Locator") diff --git a/pkg/manifest/locator_test.go b/pkg/manifest/locator_test.go index b4e95350..3992044d 100644 --- a/pkg/manifest/locator_test.go +++ b/pkg/manifest/locator_test.go @@ -108,7 +108,7 @@ func TestLocationsUnmarshalJSON(t *testing.T) { Progression: extensions.Pointer(0.74), TotalProgression: extensions.Pointer(0.32), Position: extensions.Pointer[uint](42), - OtherLocations: map[string]interface{}{ + OtherLocations: map[string]any{ "other": "other-location", }, }, l) @@ -196,7 +196,7 @@ func TestLocationsJSON(t *testing.T) { Progression: extensions.Pointer(0.74), Position: extensions.Pointer[uint](42), TotalProgression: extensions.Pointer(25.32), - OtherLocations: map[string]interface{}{ + OtherLocations: map[string]any{ "other": "other-location", }, }) diff --git a/pkg/manifest/manifest.go b/pkg/manifest/manifest.go index 8b28acc2..241cba93 100644 --- a/pkg/manifest/manifest.go +++ b/pkg/manifest/manifest.go @@ -233,13 +233,13 @@ func (m Manifest) LocatorFromLink(link Link) *Locator { // TODO log [warnings] ? // https://readium.org/webpub-manifest/ // https://readium.org/webpub-manifest/schema/publication.schema.json -func ManifestFromJSON(rawJson map[string]interface{}, packaged bool) (*Manifest, error) { +func ManifestFromJSON(rawJson map[string]any, packaged bool) (*Manifest, error) { if rawJson == nil { return nil, nil } // Parse links - rawLinks, ok := rawJson["links"].([]interface{}) + rawLinks, ok := rawJson["links"].([]any) var links []Link var err error if ok { @@ -261,7 +261,7 @@ func ManifestFromJSON(rawJson map[string]interface{}, packaged bool) (*Manifest, } // Metadata - rmt, ok := rawJson["metadata"].(map[string]interface{}) + rmt, ok := rawJson["metadata"].(map[string]any) if !ok { errors.New("'metadata' JSON object is required") } @@ -295,10 +295,10 @@ func ManifestFromJSON(rawJson map[string]interface{}, packaged bool) (*Manifest, manifest.Links = links // ReadingOrder - readingOrderRaw, ok := rawJson["readingOrder"].([]interface{}) + readingOrderRaw, ok := rawJson["readingOrder"].([]any) if !ok { // [readingOrder] used to be [spine], so we parse [spine] as a fallback. - readingOrderRaw, ok = rawJson["spine"].([]interface{}) + readingOrderRaw, ok = rawJson["spine"].([]any) if !ok { return nil, errors.New("Manifest has no valid 'readingOrder' or 'spine'") } @@ -316,7 +316,7 @@ func ManifestFromJSON(rawJson map[string]interface{}, packaged bool) (*Manifest, } // Resources - resourcesRaw, ok := rawJson["resources"].([]interface{}) + resourcesRaw, ok := rawJson["resources"].([]any) if ok { resources, err := LinksFromJSONArray(resourcesRaw) if err != nil { @@ -332,7 +332,7 @@ func ManifestFromJSON(rawJson map[string]interface{}, packaged bool) (*Manifest, } // TOC - tocRaw, ok := rawJson["toc"].([]interface{}) + tocRaw, ok := rawJson["toc"].([]any) if ok { toc, err := LinksFromJSONArray(tocRaw) if err != nil { @@ -361,7 +361,7 @@ func ManifestFromJSON(rawJson map[string]interface{}, packaged bool) (*Manifest, } func (m *Manifest) UnmarshalJSON(b []byte) error { - var object map[string]interface{} + var object map[string]any err := json.Unmarshal(b, &object) if err != nil { return err @@ -374,8 +374,8 @@ func (m *Manifest) UnmarshalJSON(b []byte) error { return nil } -func (m Manifest) ToMap(selfLink *Link) map[string]interface{} { - res := make(map[string]interface{}) +func (m Manifest) ToMap(selfLink *Link) map[string]any { + res := make(map[string]any) if len(m.Context) > 1 { res["@context"] = m.Context } else if len(m.Context) == 1 { diff --git a/pkg/manifest/manifest_test.go b/pkg/manifest/manifest_test.go index b99bff65..9c38f92f 100644 --- a/pkg/manifest/manifest_test.go +++ b/pkg/manifest/manifest_test.go @@ -72,7 +72,7 @@ func TestManifestUnmarshalFullJSON(t *testing.T) { }, Subcollections: PublicationCollectionMap{ "sub": {{ - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, Links: []Link{{Href: MustNewHREFFromString("sublink", false)}}, }}, }, @@ -239,7 +239,7 @@ func TestManifestFullJSON(t *testing.T) { }, Subcollections: PublicationCollectionMap{ "sub": {{ - Metadata: map[string]interface{}{}, + Metadata: map[string]any{}, Links: []Link{{Href: MustNewHREFFromString("sublink", false)}}, }}, }, @@ -272,7 +272,7 @@ func TestManifestFullJSON(t *testing.T) { } func TestManifestSelfLinkReplacedWhenPackaged(t *testing.T) { - var rm map[string]interface{} + var rm map[string]any require.NoError(t, json.Unmarshal([]byte(`{ "metadata": {"title": "Title"}, "links": [ @@ -295,7 +295,7 @@ func TestManifestSelfLinkReplacedWhenPackaged(t *testing.T) { } func TestManifestSelfLinkKeptWhenRemote(t *testing.T) { - var rm map[string]interface{} + var rm map[string]any require.NoError(t, json.Unmarshal([]byte(`{ "metadata": {"title": "Title"}, "links": [ @@ -318,7 +318,7 @@ func TestManifestSelfLinkKeptWhenRemote(t *testing.T) { } func TestManifestHrefResolvedToRoot(t *testing.T) { - var rm map[string]interface{} + var rm map[string]any require.NoError(t, json.Unmarshal([]byte(`{ "metadata": {"title": "Title"}, "links": [ @@ -337,7 +337,7 @@ func TestManifestHrefResolvedToRoot(t *testing.T) { } func TestManifestHrefResolvedToRootRemotePackage(t *testing.T) { - var rm map[string]interface{} + var rm map[string]any require.NoError(t, json.Unmarshal([]byte(`{ "metadata": {"title": "Title"}, "links": [ diff --git a/pkg/manifest/metadata.go b/pkg/manifest/metadata.go index 7b375025..081ce947 100644 --- a/pkg/manifest/metadata.go +++ b/pkg/manifest/metadata.go @@ -2,6 +2,7 @@ package manifest import ( "encoding/json" + "maps" "strings" "time" @@ -58,7 +59,7 @@ type Metadata struct { BelongsTo map[string]Collections `json:"belongsTo,omitempty"` MediaOverlay *MediaOverlay `json:"mediaOverlay,omitempty"` - OtherMetadata map[string]interface{} `json:"-"` // Extension point for other metadata. + OtherMetadata map[string]any `json:"-"` // Extension point for other metadata. } func (m Metadata) Title() string { @@ -128,7 +129,7 @@ const InferredAccessibilityMetadataKey = "https://readium.org/webpub-manifest#in // manifest and stored in OtherMetadata. func (m Metadata) InferredAccessibility() *A11y { var a11y *A11y - if a11yJSON, ok := m.OtherMetadata[InferredAccessibilityMetadataKey].(map[string]interface{}); ok { + if a11yJSON, ok := m.OtherMetadata[InferredAccessibilityMetadataKey].(map[string]any); ok { a11y, _ = A11yFromJSON(a11yJSON) } return a11y @@ -136,7 +137,7 @@ func (m Metadata) InferredAccessibility() *A11y { // SetOtherMetadata marshalls the value to a JSON map before storing it in // OtherMetadata under the given key. -func (m Metadata) SetOtherMetadata(key string, value interface{}) error { +func (m Metadata) SetOtherMetadata(key string, value any) error { value, err := toJSONMap(value) if err != nil { return err @@ -145,7 +146,7 @@ func (m Metadata) SetOtherMetadata(key string, value interface{}) error { return nil } -func toJSONMap(value interface{}) (map[string]interface{}, error) { +func toJSONMap(value any) (map[string]any, error) { if value, ok := value.(util.JSONMappable); ok { return value.JSONMap() } @@ -154,7 +155,7 @@ func toJSONMap(value interface{}) (map[string]interface{}, error) { if err != nil { return nil, err } - var object map[string]interface{} + var object map[string]any err = json.Unmarshal(bytes, &object) if err != nil { return nil, err @@ -162,7 +163,7 @@ func toJSONMap(value interface{}) (map[string]interface{}, error) { return object, nil } -func MetadataFromJSON(rawJson map[string]interface{}) (*Metadata, error) { +func MetadataFromJSON(rawJson map[string]any) (*Metadata, error) { if rawJson == nil { return nil, nil } @@ -184,7 +185,7 @@ func MetadataFromJSON(rawJson map[string]interface{}) (*Metadata, error) { // Accessibility var a11y *A11y - if a11yJSON, ok := rawJson["accessibility"].(map[string]interface{}); ok { + if a11yJSON, ok := rawJson["accessibility"].(map[string]any); ok { a11y, err = A11yFromJSON(a11yJSON) if err != nil { return nil, errors.Wrap(err, "failed parsing 'accessibility'") @@ -193,7 +194,7 @@ func MetadataFromJSON(rawJson map[string]interface{}) (*Metadata, error) { // TDMRep (Text & Data Mining Reservation Protocol) var tdm *TDM - if tdmJSON, ok := rawJson["tdm"].(map[string]interface{}); ok { + if tdmJSON, ok := rawJson["tdm"].(map[string]any); ok { tdm, err = TDMFromJSON(tdmJSON) if err != nil { return nil, errors.Wrap(err, "failed parsing 'tdm'") @@ -367,9 +368,9 @@ func MetadataFromJSON(rawJson map[string]interface{}) (*Metadata, error) { } // BelongsTo - belongsToRaw, ok := rawJson["belongsTo"].(map[string]interface{}) + belongsToRaw, ok := rawJson["belongsTo"].(map[string]any) if !ok { - belongsToRaw, _ = rawJson["belongs_to"].(map[string]interface{}) + belongsToRaw, _ = rawJson["belongs_to"].(map[string]any) } if belongsToRaw != nil { belongsTo := make(map[string]Collections) @@ -387,7 +388,7 @@ func MetadataFromJSON(rawJson map[string]interface{}) (*Metadata, error) { } // Media Overlay - if mediaOverlay, ok := rawJson["mediaOverlay"].(map[string]interface{}); ok { + if mediaOverlay, ok := rawJson["mediaOverlay"].(map[string]any); ok { metadata.MediaOverlay = &MediaOverlay{} decoder, _ := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ @@ -448,7 +449,7 @@ func MetadataFromJSON(rawJson map[string]interface{}) (*Metadata, error) { } func (m *Metadata) UnmarshalJSON(b []byte) error { - var object map[string]interface{} + var object map[string]any err := json.Unmarshal(b, &object) if err != nil { return err @@ -465,11 +466,9 @@ func (m *Metadata) UnmarshalJSON(b []byte) error { var ToolkitVersionKey = "https://github.com/readium/go-toolkit#version" func (m Metadata) MarshalJSON() ([]byte, error) { - j := make(map[string]interface{}) + j := make(map[string]any) if m.OtherMetadata != nil { - for k, v := range m.OtherMetadata { - j[k] = v - } + maps.Copy(j, m.OtherMetadata) } if ToolkitVersionKey != "" { diff --git a/pkg/manifest/metadata_test.go b/pkg/manifest/metadata_test.go index 9d5525b0..17bc639e 100644 --- a/pkg/manifest/metadata_test.go +++ b/pkg/manifest/metadata_test.go @@ -138,9 +138,9 @@ func TestMetadataUnmarshalFullJSON(t *testing.T) { "collection": {{LocalizedName: NewLocalizedStringFromString("Collection")}}, "series": {{LocalizedName: NewLocalizedStringFromString("Series")}}, }, - OtherMetadata: map[string]interface{}{ + OtherMetadata: map[string]any{ "other-metadata1": "value", - "other-metadata2": []interface{}{float64(42)}, + "other-metadata2": []any{float64(42)}, }, }, m, "parsed JSON object should be equal to Metadata object") } @@ -277,9 +277,9 @@ func TestMetadataFullJSON(t *testing.T) { "collection": {{LocalizedName: NewLocalizedStringFromString("Collection")}}, "series": {{LocalizedName: NewLocalizedStringFromString("Series")}}, }, - OtherMetadata: map[string]interface{}{ + OtherMetadata: map[string]any{ "other-metadata1": "value", - "other-metadata2": []interface{}{float64(42)}, + "other-metadata2": []any{float64(42)}, }, }) require.NoError(t, err) diff --git a/pkg/manifest/profile.go b/pkg/manifest/profile.go index a07533fc..955d56bf 100644 --- a/pkg/manifest/profile.go +++ b/pkg/manifest/profile.go @@ -27,7 +27,7 @@ type Profiles []Profile // Convert to string slice func (p Profiles) toss() []string { - return interface{}(p).([]string) // Ugly... + return any(p).([]string) // Ugly... } func (p Profiles) MarshalJSON() ([]byte, error) { diff --git a/pkg/manifest/properties.go b/pkg/manifest/properties.go index 08b455ed..065a555f 100644 --- a/pkg/manifest/properties.go +++ b/pkg/manifest/properties.go @@ -7,7 +7,7 @@ import ( ) // Properties associated with a linked resource -type Properties map[string]interface{} +type Properties map[string]any // Properties should be immutable, therefore these functions have been removed. // The code is left here in case it's useful in a future implementation. @@ -30,7 +30,7 @@ func (p *Properties) Delete(key string) Properties { return *p }*/ -func (p *Properties) Get(key string) interface{} { +func (p *Properties) Get(key string) any { if p != nil { return (*p)[key] } @@ -90,7 +90,7 @@ func (p Properties) Encryption() *Encryption { if v == nil { return nil } - mp, ok := v.(map[string]interface{}) + mp, ok := v.(map[string]any) if mp == nil || !ok { return nil } @@ -119,7 +119,7 @@ func (p Properties) Hash() HashList { if v == nil { return nil } - cv, ok := v.([]interface{}) + cv, ok := v.([]any) if !ok { return nil } @@ -130,12 +130,12 @@ func (p Properties) Hash() HashList { return hashes } -func PropertiesFromJSON(rawJson interface{}) (Properties, error) { +func PropertiesFromJSON(rawJson any) (Properties, error) { if rawJson == nil { return make(Properties), nil } - properties, ok := rawJson.(map[string]interface{}) + properties, ok := rawJson.(map[string]any) if !ok { return nil, errors.New("Properties has invalid JSON object") } @@ -143,7 +143,7 @@ func PropertiesFromJSON(rawJson interface{}) (Properties, error) { } func (p *Properties) UnmarshalJSON(data []byte) error { - var d interface{} + var d any err := json.Unmarshal(data, &d) if err != nil { return err diff --git a/pkg/manifest/properties_hash.go b/pkg/manifest/properties_hash.go index 21b883f3..479d77e4 100644 --- a/pkg/manifest/properties_hash.go +++ b/pkg/manifest/properties_hash.go @@ -75,10 +75,10 @@ func (h *HashList) Deduplicate() { *h = unique } -func HashListFromJSONArray(rawJsonArray []interface{}) (HashList, error) { +func HashListFromJSONArray(rawJsonArray []any) (HashList, error) { var hashes HashList for _, item := range rawJsonArray { - itemMap, ok := item.(map[string]interface{}) + itemMap, ok := item.(map[string]any) if !ok { return nil, errors.Errorf("invalid hash item: %v", item) } @@ -91,10 +91,10 @@ func HashListFromJSONArray(rawJsonArray []interface{}) (HashList, error) { return hashes, nil } -func (h HashList) ToJSONArray() []interface{} { - jsonArray := make([]interface{}, 0, len(h)) +func (h HashList) ToJSONArray() []any { + jsonArray := make([]any, 0, len(h)) for _, hash := range h { - jsonArray = append(jsonArray, map[string]interface{}{ + jsonArray = append(jsonArray, map[string]any{ "algorithm": hash.Algorithm, "value": hash.Value, }) diff --git a/pkg/manifest/properties_test.go b/pkg/manifest/properties_test.go index c75dfebd..398dffc4 100644 --- a/pkg/manifest/properties_test.go +++ b/pkg/manifest/properties_test.go @@ -29,7 +29,7 @@ func TestPropertiesUnmarshalFullJSON(t *testing.T) { assert.Equal(t, Properties{ "other-property1": "value", - "other-property2": []interface{}{float64(42)}, + "other-property2": []any{float64(42)}, }, p) } diff --git a/pkg/manifest/subject.go b/pkg/manifest/subject.go index 855ce4ad..fa480ca2 100644 --- a/pkg/manifest/subject.go +++ b/pkg/manifest/subject.go @@ -31,7 +31,7 @@ func (s Subject) SortAs() string { // Parses a [Subject] from its RWPM JSON representation. // A subject can be parsed from a single string, or a full-fledged object. // The [links]' href and their children's will be normalized recursively using the provided [normalizeHref] closure. -func SubjectFromJSON(rawJson interface{}) (*Subject, error) { +func SubjectFromJSON(rawJson any) (*Subject, error) { if rawJson == nil { return nil, nil } @@ -42,7 +42,7 @@ func SubjectFromJSON(rawJson interface{}) (*Subject, error) { return nil, errors.Wrap(err, "failed parsing Subject as LocalizedString") } return &Subject{LocalizedName: *localizedName}, nil - case map[string]interface{}: + case map[string]any: localizedName, err := LocalizedStringFromJSON(rjs["name"]) if err != nil { return nil, errors.Wrap(err, "failed parsing Subject 'name' as LocalizedString") @@ -65,7 +65,7 @@ func SubjectFromJSON(rawJson interface{}) (*Subject, error) { } // links - lln, ok := rjs["links"].([]interface{}) + lln, ok := rjs["links"].([]any) if ok { links, err := LinksFromJSONArray(lln) if err != nil { @@ -82,10 +82,10 @@ func SubjectFromJSON(rawJson interface{}) (*Subject, error) { // Creates a list of [Subject] from its RWPM JSON representation. // The [links]' href and their children's will be normalized recursively using the provided [normalizeHref] closure. -func SubjectFromJSONArray(rawJsonArray interface{}) ([]Subject, error) { +func SubjectFromJSONArray(rawJsonArray any) ([]Subject, error) { var subjects []Subject switch rjx := rawJsonArray.(type) { - case []interface{}: + case []any: subjects = make([]Subject, 0, len(rjx)) for i, entry := range rjx { rs, err := SubjectFromJSON(entry) @@ -110,7 +110,7 @@ func SubjectFromJSONArray(rawJsonArray interface{}) ([]Subject, error) { } func (s *Subject) UnmarshalJSON(data []byte) error { - var object interface{} + var object any err := json.Unmarshal(data, &object) if err != nil { return err diff --git a/pkg/manifest/tdm.go b/pkg/manifest/tdm.go index 7f464eb9..f7d4b86e 100644 --- a/pkg/manifest/tdm.go +++ b/pkg/manifest/tdm.go @@ -29,7 +29,7 @@ func (t TDMReservation) String() string { return string(t) } -func TDMFromJSON(rawJSON map[string]interface{}) (*TDM, error) { +func TDMFromJSON(rawJSON map[string]any) (*TDM, error) { if rawJSON == nil { return nil, nil } @@ -52,13 +52,13 @@ func TDMFromJSON(rawJSON map[string]interface{}) (*TDM, error) { } func (t *TDM) UnmarshalJSON(data []byte) error { - var d interface{} + var d any err := json.Unmarshal(data, &d) if err != nil { return err } - mp, ok := d.(map[string]interface{}) + mp, ok := d.(map[string]any) if !ok { return errors.New("tdm object not a map with string keys") } diff --git a/pkg/manifest/tdm_test.go b/pkg/manifest/tdm_test.go index 9502a728..168d0d4a 100644 --- a/pkg/manifest/tdm_test.go +++ b/pkg/manifest/tdm_test.go @@ -9,7 +9,7 @@ import ( ) func TestTDMFromJSON(t *testing.T) { - rawJSON := map[string]interface{}{ + rawJSON := map[string]any{ "policy": "https://provider.com/policies/policy.json", "reservation": "all", } @@ -19,7 +19,7 @@ func TestTDMFromJSON(t *testing.T) { assert.Equal(t, "https://provider.com/policies/policy.json", tdm.Policy) assert.Equal(t, TDMReservationAll, tdm.Reservation) - rawJSON = map[string]interface{}{ + rawJSON = map[string]any{ "reservation": "none", } tdm, err = TDMFromJSON(rawJSON) diff --git a/pkg/manifest/utils.go b/pkg/manifest/utils.go index 68f72c6d..ec3cde38 100644 --- a/pkg/manifest/utils.go +++ b/pkg/manifest/utils.go @@ -2,16 +2,17 @@ package manifest import ( "fmt" + "slices" "time" "github.com/readium/go-toolkit/pkg/internal/extensions" ) -func parseSliceOrString(value interface{}, deduplicate bool) (result []string, err error) { +func parseSliceOrString(value any, deduplicate bool) (result []string, err error) { switch v := value.(type) { case string: result = []string{v} // Just a single item - case []interface{}: + case []any: result = make([]string, 0, len(v)) for i, vv := range v { str, ok := vv.(string) @@ -58,10 +59,8 @@ func nilboolEq(source *bool, val bool) bool { func firstLinkWithRel(links []Link, rel string) *Link { for _, link := range links { - for _, linkRel := range link.Rels { - if linkRel == rel { - return &link - } + if slices.Contains(link.Rels, rel) { + return &link } } return nil @@ -70,7 +69,7 @@ func firstLinkWithRel(links []Link, rel string) *Link { // Utilities for convenient JSON unmarshalling // TODO replace a lot of these with generics! -func parseOptTime(raw interface{}) *time.Time { +func parseOptTime(raw any) *time.Time { rt, ok := raw.(string) if !ok { return nil @@ -80,24 +79,24 @@ func parseOptTime(raw interface{}) *time.Time { return t } -func parseOptString(raw interface{}) string { +func parseOptString(raw any) string { rs, _ := raw.(string) return rs } -func parseOptBool(raw interface{}) bool { +func parseOptBool(raw any) bool { rb, _ := raw.(bool) return rb } -func parseOptFloat64(raw interface{}) float64 { +func parseOptFloat64(raw any) float64 { rb, _ := raw.(float64) return rb } // parseOptInt64 reads an integer that may come from JSON (float64) or from // in-memory maps built with native integer types. -func parseOptInt64(raw interface{}) int64 { +func parseOptInt64(raw any) int64 { switch v := raw.(type) { case float64: return int64(v) diff --git a/pkg/mediatype/sniffer.go b/pkg/mediatype/sniffer.go index 0d4ea304..7bf3618b 100644 --- a/pkg/mediatype/sniffer.go +++ b/pkg/mediatype/sniffer.go @@ -227,7 +227,7 @@ func SniffWebpub(ctx context.Context, context SnifferContext) *MediaType { if !isRWPMJSON(rwpm) { rwpm = nil if bin := context.ReadArchiveEntryAt(ctx, "manifest.json"); bin != nil { - var js map[string]interface{} + var js map[string]any if json.Unmarshal(bin, &js) == nil && isRWPMJSON(js) { isManifest = false rwpm = js @@ -277,7 +277,7 @@ func SniffWebpub(ctx context.Context, context SnifferContext) *MediaType { func SniffW3CWPUB(ctx context.Context, context SnifferContext) *MediaType { if js := context.ContentAsJSON(); js != nil { if ctx, ok := js["@context"]; ok { - if context, ok := ctx.([]interface{}); ok { + if context, ok := ctx.([]any); ok { for _, v := range context { if val, ok := v.(string); ok { if val == "https://www.w3.org/ns/wp-context" { @@ -321,10 +321,10 @@ func SniffLPF(ctx context.Context, context SnifferContext) *MediaType { } if entry := context.ReadArchiveEntryAt(ctx, "publication.json"); entry != nil { - var js map[string]interface{} + var js map[string]any if err := json.Unmarshal(entry, &js); err == nil && js != nil { if ctx, ok := js["@context"]; ok { - if context, ok := ctx.([]interface{}); ok { + if context, ok := ctx.([]any); ok { for _, v := range context { if val, ok := v.(string); ok { if val == "https://www.w3.org/ns/pub-context" { diff --git a/pkg/mediatype/sniffer_context.go b/pkg/mediatype/sniffer_context.go index e2e6eb0d..209f25b3 100644 --- a/pkg/mediatype/sniffer_context.go +++ b/pkg/mediatype/sniffer_context.go @@ -6,6 +6,7 @@ import ( "encoding/xml" "errors" "io" + "slices" "strings" "github.com/readium/go-toolkit/pkg/archive" @@ -25,7 +26,7 @@ type SnifferContext struct { _loadedContentAsString bool _contentAsXML *XMLNode _loadedContentAsXML bool - _contentAsJSON map[string]interface{} + _contentAsJSON map[string]any _loadedContentAsJSON bool _contentAsArchive archive.Archive _contentAsArchiveErr error @@ -73,10 +74,8 @@ func (s SnifferContext) HasFileExtension(fileExtensions ...string) bool { selfExtensions := s.FileExtensions() for _, fileExtension := range fileExtensions { lowerExt := strings.ToLower(fileExtension) - for _, fext := range selfExtensions { - if fext == lowerExt { - return true - } + if slices.Contains(selfExtensions, lowerExt) { + return true } } return false @@ -200,14 +199,14 @@ func (s *SnifferContext) Close() { var errClosedSnifferContext = errors.New("the sniffing context is closed") // Content parsed as generic JSON interface. -func (s SnifferContext) ContentAsJSON() map[string]interface{} { +func (s SnifferContext) ContentAsJSON() map[string]any { if !s._loadedContentAsJSON { s._loadedContentAsJSON = true stream := s.Stream() if stream == nil { return nil // No stream } - var jd map[string]interface{} + var jd map[string]any err := json.NewDecoder(stream).Decode(&jd) if err != nil { return nil diff --git a/pkg/mediatype/sniffer_rwpm.go b/pkg/mediatype/sniffer_rwpm.go index d404f9fd..1232f3df 100644 --- a/pkg/mediatype/sniffer_rwpm.go +++ b/pkg/mediatype/sniffer_rwpm.go @@ -21,15 +21,15 @@ const lcpSchemeURI = "http://readium.org/2014/01/lcp" // string or a localized-string object whose translations are all strings, as // [manifest.LocalizedStringFromJSON] requires. This is the minimal RWPM signature, // shared by Readium manifests and OPDS 2 documents. -func rwpmHasMetadataTitle(js map[string]interface{}) bool { - metadata, ok := js["metadata"].(map[string]interface{}) +func rwpmHasMetadataTitle(js map[string]any) bool { + metadata, ok := js["metadata"].(map[string]any) if !ok { return false } switch title := metadata["title"].(type) { case string: // Plain title return true - case map[string]interface{}: // Localized title: every translation must be a string + case map[string]any: // Localized title: every translation must be a string for _, v := range title { if _, ok := v.(string); !ok { return false @@ -44,7 +44,7 @@ func rwpmHasMetadataTitle(js map[string]interface{}) bool { // Returns whether the JSON has the shape of a RWPM that [manifest.ManifestFromJSON] // would accept: a `metadata` object bearing a title, along with a `readingOrder` // (or legacy `spine`) array of link objects which have an `href` and a valid `type`. -func isRWPMJSON(js map[string]interface{}) bool { +func isRWPMJSON(js map[string]any) bool { if js == nil { return false } @@ -53,14 +53,14 @@ func isRWPMJSON(js map[string]interface{}) bool { return false } - rawLinks, ok := js["readingOrder"].([]interface{}) + rawLinks, ok := js["readingOrder"].([]any) if !ok { - if rawLinks, ok = js["spine"].([]interface{}); !ok { + if rawLinks, ok = js["spine"].([]any); !ok { return false } } for _, raw := range rawLinks { - link, ok := raw.(map[string]interface{}) + link, ok := raw.(map[string]any) if !ok { return false } @@ -79,14 +79,14 @@ func isRWPMJSON(js map[string]interface{}) bool { } // Returns the link objects of the given collection ("readingOrder", "resources", "links"...). -func rwpmLinks(js map[string]interface{}, key string) []map[string]interface{} { - raw, ok := js[key].([]interface{}) +func rwpmLinks(js map[string]any, key string) []map[string]any { + raw, ok := js[key].([]any) if !ok { return nil } - links := make([]map[string]interface{}, 0, len(raw)) + links := make([]map[string]any, 0, len(raw)) for _, v := range raw { - if link, ok := v.(map[string]interface{}); ok { + if link, ok := v.(map[string]any); ok { links = append(links, link) } } @@ -95,7 +95,7 @@ func rwpmLinks(js map[string]interface{}, key string) []map[string]interface{} { // Returns the media types of the reading order (or legacy spine) links. // Links without a valid media type are skipped, like [manifest.ManifestFromJSON] drops them. -func rwpmReadingOrderTypes(js map[string]interface{}) []MediaType { +func rwpmReadingOrderTypes(js map[string]any) []MediaType { links := rwpmLinks(js, "readingOrder") if links == nil { links = rwpmLinks(js, "spine") @@ -118,19 +118,19 @@ func rwpmReadingOrderTypes(js map[string]interface{}) []MediaType { // Returns whether the RWPM conforms to the given profile, mirroring // [manifest.Manifest.ConformsTo]: either the profile is declared in // `metadata.conformsTo`, or every reading order resource satisfies [matches]. -func rwpmConformsTo(js map[string]interface{}, profile string, matches func(mt MediaType) bool) bool { +func rwpmConformsTo(js map[string]any, profile string, matches func(mt MediaType) bool) bool { types := rwpmReadingOrderTypes(js) if len(types) == 0 { return false } - metadata, _ := js["metadata"].(map[string]interface{}) + metadata, _ := js["metadata"].(map[string]any) switch conformsTo := metadata["conformsTo"].(type) { case string: if conformsTo == profile { return true } - case []interface{}: + case []any: for _, v := range conformsTo { if v == profile { return true @@ -147,11 +147,11 @@ func rwpmConformsTo(js map[string]interface{}, profile string, matches func(mt M } // Returns whether the `self` link of the RWPM matches one of the given media types. -func rwpmSelfLinkMatches(js map[string]interface{}, mediaTypes ...*MediaType) bool { +func rwpmSelfLinkMatches(js map[string]any, mediaTypes ...*MediaType) bool { for _, link := range rwpmLinks(js, "links") { - rels, ok := link["rel"].([]interface{}) + rels, ok := link["rel"].([]any) if !ok { - rels = []interface{}{link["rel"]} + rels = []any{link["rel"]} } isSelf := false for _, rel := range rels { @@ -180,11 +180,11 @@ func rwpmSelfLinkMatches(js map[string]interface{}, mediaTypes ...*MediaType) bo // Returns whether a link of the RWPM's `links` collection has a relation starting with // the given prefix. -func rwpmHasLinkWithRelPrefix(js map[string]interface{}, prefix string) bool { +func rwpmHasLinkWithRelPrefix(js map[string]any, prefix string) bool { for _, link := range rwpmLinks(js, "links") { - rels, ok := link["rel"].([]interface{}) + rels, ok := link["rel"].([]any) if !ok { - rels = []interface{}{link["rel"]} + rels = []any{link["rel"]} } for _, rel := range rels { if s, ok := rel.(string); ok && strings.HasPrefix(s, prefix) { @@ -196,17 +196,17 @@ func rwpmHasLinkWithRelPrefix(js map[string]interface{}, prefix string) bool { } // Returns whether a reading order resource of the RWPM is encrypted with the LCP scheme. -func rwpmHasLCPScheme(js map[string]interface{}) bool { +func rwpmHasLCPScheme(js map[string]any) bool { links := rwpmLinks(js, "readingOrder") if links == nil { links = rwpmLinks(js, "spine") } for _, link := range links { - properties, ok := link["properties"].(map[string]interface{}) + properties, ok := link["properties"].(map[string]any) if !ok { continue } - encrypted, ok := properties["encrypted"].(map[string]interface{}) + encrypted, ok := properties["encrypted"].(map[string]any) if !ok { continue } diff --git a/pkg/parser/audio/duration_formats.go b/pkg/parser/audio/duration_formats.go index 5aafe0a8..b5ad1bb0 100644 --- a/pkg/parser/audio/duration_formats.go +++ b/pkg/parser/audio/duration_formats.go @@ -363,10 +363,7 @@ const ( // the Duration element from the Segment Info, scaled by the TimecodeScale. func probeWebMDuration(ctx context.Context, res fetcher.Resource, size int64) float64 { // The Segment Info element is normally near the beginning of the file. - readLen := int64(1 << 20) - if readLen > size { - readLen = size - } + readLen := min(int64(1<<20), size) buf := readRange(ctx, res, 0, readLen) if len(buf) < 4 || !bytes.HasPrefix(buf, []byte{0x1A, 0x45, 0xDF, 0xA3}) { return 0 @@ -500,10 +497,7 @@ func probeAACDuration(ctx context.Context, res fetcher.Resource, size int64) flo return 0 } remaining := size - start - readLen := remaining - if readLen > aacScanWindow { - readLen = aacScanWindow - } + readLen := min(remaining, aacScanWindow) data := readRange(ctx, res, start, readLen) if len(data) < 7 { return 0 diff --git a/pkg/parser/audio/formats_test.go b/pkg/parser/audio/formats_test.go index ed259c36..578d5bb0 100644 --- a/pkg/parser/audio/formats_test.go +++ b/pkg/parser/audio/formats_test.go @@ -137,7 +137,7 @@ func TestParseTimecode(t *testing.T) { } func TestVorbisChapters(t *testing.T) { - tags := &audioTags{Raw: map[string]interface{}{ + tags := &audioTags{Raw: map[string]any{ "CHAPTER000": "00:00:00.000", "CHAPTER000NAME": "Opening", "CHAPTER001": "00:01:30.000", diff --git a/pkg/parser/audio/mp4.go b/pkg/parser/audio/mp4.go index e19a8ca4..4a77f693 100644 --- a/pkg/parser/audio/mp4.go +++ b/pkg/parser/audio/mp4.go @@ -358,7 +358,7 @@ func parseNeroChapters(b []byte) []chapterEntry { pos++ chapters := make([]chapterEntry, 0, count) - for c := 0; c < count; c++ { + for range count { if pos+9 > len(b) { break } diff --git a/pkg/parser/audio/ogg.go b/pkg/parser/audio/ogg.go index 63dad86b..09eb5829 100644 --- a/pkg/parser/audio/ogg.go +++ b/pkg/parser/audio/ogg.go @@ -31,10 +31,7 @@ func probeOggDuration(ctx context.Context, res fetcher.Resource, size int64) flo } // Read a window at the end of the file large enough to contain the last page. - tailLen := int64(65536) - if tailLen > size { - tailLen = size - } + tailLen := min(int64(65536), size) tail, err := res.Read(ctx, size-tailLen, size-1) if err != nil || len(tail) == 0 { return 0 diff --git a/pkg/parser/audio/readcache.go b/pkg/parser/audio/readcache.go index 4666813f..92ceefb7 100644 --- a/pkg/parser/audio/readcache.go +++ b/pkg/parser/audio/readcache.go @@ -144,10 +144,7 @@ func (c *readCache) fetch(ctx context.Context, first, last int64) *fetcher.Resou c.blocks[bi] = []byte{} continue } - hi := lo + c.blockSize - if hi > int64(len(data)) { - hi = int64(len(data)) - } + hi := min(lo+c.blockSize, int64(len(data))) c.blocks[bi] = data[lo:hi] } c.mu.Unlock() diff --git a/pkg/parser/epub/deobfuscator.go b/pkg/parser/epub/deobfuscator.go index 19f7ab13..7c7331f2 100644 --- a/pkg/parser/epub/deobfuscator.go +++ b/pkg/parser/epub/deobfuscator.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/readium/go-toolkit/pkg/archive" "github.com/readium/go-toolkit/pkg/fetcher" + "github.com/readium/go-toolkit/pkg/internal/obfuscation" ) var algorithm2length = map[string]int64{ @@ -245,8 +246,5 @@ func deobfuscateFont(data []byte, start int64, obfuscationKey []byte, obfuscatio if max > dlen { max = dlen } - olen := int64(len(obfuscationKey)) - for i := int64(0); i < max; i++ { - data[i] ^= obfuscationKey[(start+i)%olen] - } + obfuscation.XOR(data[:max], obfuscationKey, start) } diff --git a/pkg/parser/epub/deobfuscator_test.go b/pkg/parser/epub/deobfuscator_test.go index 7a8aa9c0..aceec3db 100644 --- a/pkg/parser/epub/deobfuscator_test.go +++ b/pkg/parser/epub/deobfuscator_test.go @@ -2,6 +2,7 @@ package epub import ( "bytes" + "fmt" "testing" "github.com/readium/go-toolkit/pkg/fetcher" @@ -30,7 +31,7 @@ func withDeobfuscator(t *testing.T, href string, algorithm string, start, end in } if algorithm != "" { link.Properties = manifest.Properties{ - "encrypted": map[string]interface{}{ + "encrypted": map[string]any{ "algorithm": algorithm, }, } @@ -77,6 +78,25 @@ func TestDeobfuscatorAdobe(t *testing.T) { }) } +func TestDeobfuscatorRanges(t *testing.T) { + for _, algorithm := range []struct { + name string + href string + uri string + }{ + {"IDPF", "deobfuscation/cut-cut.obf.woff", "http://www.idpf.org/2008/embedding"}, + {"Adobe", "deobfuscation/cut-cut.adb.woff", "http://ns.adobe.com/pdf/enc#RC"}, + } { + for _, span := range [][2]int64{{1, 2}, {13, 31}, {13, 2000}, {1023, 2000}, {1024, 2000}, {1039, 2000}, {1040, 2000}} { + t.Run(fmt.Sprintf("%s/%d-%d", algorithm.name, span[0], span[1]), func(t *testing.T) { + withDeobfuscator(t, algorithm.href, algorithm.uri, span[0], span[1], func(clean, obfu []byte) { + assert.Equal(t, clean, obfu) + }) + }) + } + } +} + func TestDeobfuscatorNoAlgorithm(t *testing.T) { withDeobfuscator(t, "deobfuscation/cut-cut.woff", "", 0, 0, func(clean, obfu []byte) { assert.Equal(t, clean, obfu) diff --git a/pkg/parser/epub/factory.go b/pkg/parser/epub/factory.go index d3632820..fda889f5 100644 --- a/pkg/parser/epub/factory.go +++ b/pkg/parser/epub/factory.go @@ -1,6 +1,8 @@ package epub import ( + "maps" + "github.com/readium/go-toolkit/pkg/internal/extensions" "github.com/readium/go-toolkit/pkg/manifest" ) @@ -16,6 +18,7 @@ type PublicationFactory struct { pubMetadata PubMetadataAdapter itemrefByIdref map[string]ItemRef itemMetadata map[string]LinkMetadataAdapter + coverID string } func (f PublicationFactory) Create() manifest.Manifest { @@ -35,20 +38,25 @@ func (f PublicationFactory) Create() manifest.Manifest { readingProgression: spine.direction, displayOptions: f.DisplayOptions, } - f.itemMetadata = make(map[string]LinkMetadataAdapter) + f.coverID = f.pubMetadata.Cover() + f.itemMetadata = make(map[string]LinkMetadataAdapter, len(f.PackageDocument.metadata.refine)) for k, v := range f.PackageDocument.metadata.refine { f.itemMetadata[k] = LinkMetadataAdapter{ epubVersion: epubVersion, items: v, } } - f.itemById = make(map[string]Item) + f.itemById = make(map[string]Item, len(mani)) for _, item := range mani { f.itemById[item.ID] = item } - f.itemrefByIdref = make(map[string]ItemRef) + f.itemrefByIdref = make(map[string]ItemRef, len(spine.itemrefs)) + linearCount := 0 for _, v := range spine.itemrefs { f.itemrefByIdref[v.idref] = v + if v.linear { + linearCount++ + } } // Compute Metadata @@ -60,28 +68,31 @@ func (f PublicationFactory) Create() manifest.Manifest { } // Compute Links - var readingOrderIds []string + readingOrderIDs := make(map[string]struct{}, linearCount) + readingOrder := make(manifest.LinkList, 0, linearCount) for _, v := range spine.itemrefs { - if v.linear { - readingOrderIds = append(readingOrderIds, v.idref) + if !v.linear { + continue } - } - readingOrder := make(manifest.LinkList, 0, len(readingOrderIds)) - for _, id := range readingOrderIds { - item, ok := f.itemById[id] + readingOrderIDs[v.idref] = struct{}{} + item, ok := f.itemById[v.idref] if ok { - readingOrder = append(readingOrder, f.computeLink(item, []string{})) + readingOrder = append(readingOrder, f.computeLink(item, nil)) } } - var resourceItems []Item + // Count first to size the link slice without retaining a second copy of the + // manifest items. Membership checks must not scan the whole spine per item. + resourceCount := 0 for _, item := range mani { - if !extensions.Contains(readingOrderIds, item.ID) { - resourceItems = append(resourceItems, item) + if _, linear := readingOrderIDs[item.ID]; !linear { + resourceCount++ } } - resources := make(manifest.LinkList, 0, len(resourceItems)) - for _, item := range resourceItems { - resources = append(resources, f.computeLink(item, []string{})) + resources := make(manifest.LinkList, 0, resourceCount) + for _, item := range mani { + if _, linear := readingOrderIDs[item.ID]; !linear { + resources = append(resources, f.computeLink(item, nil)) + } } ret := manifest.Manifest{ @@ -169,12 +180,15 @@ func (f PublicationFactory) computeLink(item Item, fallbackChain []string) manif } func (f PublicationFactory) computePropertiesAndRels(item Item, itemref *ItemRef) ([]string, manifest.Properties) { - properties := make(map[string]interface{}) + var properties map[string]any var rels []string manifestRels, contains, others := parseItemProperties(item.Properties) for _, v := range manifestRels { rels = extensions.AddToSet(rels, v) } + if len(contains) > 0 || len(others) > 0 { + properties = make(map[string]any, len(others)+1) + } if len(contains) > 0 { properties["contains"] = contains } @@ -182,18 +196,25 @@ func (f PublicationFactory) computePropertiesAndRels(item Item, itemref *ItemRef properties[v] = true } if itemref != nil { - for k, v := range parseItemrefProperties(itemref.properties) { - properties[k] = v + itemrefProperties := parseItemrefProperties(itemref.properties) + if properties == nil { + properties = itemrefProperties + } else { + maps.Copy(properties, itemrefProperties) } } - coverId := f.pubMetadata.Cover() - if coverId == item.ID { + if f.coverID == item.ID { rels = extensions.AddToSet(rels, "cover") } - if edat, ok := f.EncryptionData[item.Href.Normalize().String()]; ok { - properties["encrypted"] = edat.ToMap() // ToMap makes it JSON-like + if len(f.EncryptionData) > 0 { + if edat, ok := f.EncryptionData[item.Href.Normalize().String()]; ok { + if properties == nil { + properties = make(map[string]any, 1) + } + properties["encrypted"] = edat.ToMap() // ToMap makes it JSON-like + } } return rels, manifest.Properties(properties) @@ -247,22 +268,25 @@ func parseItemProperties(properties []string) (rels []string, contains []string, return } -func parseItemrefProperties(properties []string) map[string]interface{} { - linkProperties := make(map[string]interface{}) +func parseItemrefProperties(properties []string) map[string]any { + var page string for _, property := range properties { switch property { // Page case VocabularyRendition + "page-spread-center": - linkProperties["page"] = "center" + page = "center" case VocabularyRendition + "page-spread-left": fallthrough case VocabularyItemref + "page-spread-left": - linkProperties["page"] = "left" + page = "left" case VocabularyRendition + "page-spread-right": fallthrough case VocabularyItemref + "page-spread-right": - linkProperties["page"] = "right" + page = "right" } } - return linkProperties + if page == "" { + return nil + } + return map[string]any{"page": page} } diff --git a/pkg/parser/epub/factory_performance_test.go b/pkg/parser/epub/factory_performance_test.go new file mode 100644 index 00000000..ea79dd66 --- /dev/null +++ b/pkg/parser/epub/factory_performance_test.go @@ -0,0 +1,157 @@ +package epub + +import ( + "fmt" + "testing" + + "github.com/readium/go-toolkit/pkg/fetcher" + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/readium/go-toolkit/pkg/mediatype" + "github.com/readium/go-toolkit/pkg/util/url" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// BenchmarkPublicationFactory isolates OPF-to-manifest conversion from ZIP I/O +// and XML parsing. Increasing item counts expose accidental quadratic work in +// the split between reading order and resources. +func BenchmarkPublicationFactory(b *testing.B) { + for _, size := range []int{32, 256, 2048} { + for _, encrypted := range []bool{false, true} { + b.Run(fmt.Sprintf("items=%d/encrypted=%t", size, encrypted), func(b *testing.B) { + factory := benchmarkPublicationFactory(size, encrypted) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + publication := factory.Create() + if len(publication.ReadingOrder) != size*3/4 || len(publication.Resources) != size-size*3/4 { + b.Fatal("incorrect reading order or resource count") + } + } + }) + } + } +} + +func benchmarkPublicationFactory(size int, encrypted bool) PublicationFactory { + factory := PublicationFactory{ + FallbackTitle: "Factory benchmark", + PackageDocument: PackageDocument{ + EPUBVersion: 3, + EPUBVersionString: "3.0", + Manifest: make([]Item, size), + Spine: Spine{itemrefs: make([]ItemRef, size*3/4)}, + }, + } + if encrypted { + factory.EncryptionData = make(map[string]manifest.Encryption) + } + for i := range size { + id := fmt.Sprintf("item-%04d", i) + href := url.MustURLFromString(fmt.Sprintf("OPS/chapter-%04d.xhtml", i)) + factory.PackageDocument.Manifest[i] = Item{ID: id, Href: href, MediaType: &mediatype.XHTML} + if i < len(factory.PackageDocument.Spine.itemrefs) { + factory.PackageDocument.Spine.itemrefs[i] = ItemRef{idref: id, linear: true} + } + if encrypted && i%16 == 0 { + factory.EncryptionData[href.String()] = manifest.Encryption{Algorithm: "http://www.w3.org/2001/04/xmlenc#aes256-cbc"} + } + } + return factory +} + +func TestPublicationFactoryReadingOrderMembership(t *testing.T) { + item := func(id, href string) Item { + return Item{ID: id, Href: url.MustURLFromString(href)} + } + factory := PublicationFactory{ + PackageDocument: PackageDocument{ + Manifest: []Item{ + item("chapter", "superseded.xhtml"), + item("nonlinear", "notes.xhtml"), + item("chapter", "chapter.xhtml"), + item("resource", "image.png"), + }, + Spine: Spine{itemrefs: []ItemRef{ + {idref: "chapter", linear: true}, + {idref: "missing", linear: true}, + {idref: "nonlinear", linear: false}, + {idref: "chapter", linear: true}, + }}, + }, + } + publication := factory.Create() + require.Len(t, publication.ReadingOrder, 2) + assert.Equal(t, "chapter.xhtml", publication.ReadingOrder[0].Href.String()) + assert.Equal(t, "chapter.xhtml", publication.ReadingOrder[1].Href.String()) + require.Len(t, publication.Resources, 2) + assert.Equal(t, "notes.xhtml", publication.Resources[0].Href.String()) + assert.Equal(t, "image.png", publication.Resources[1].Href.String()) +} + +func TestPublicationFactoryCombinesLinkProperties(t *testing.T) { + encryption := manifest.Encryption{Algorithm: "test-encryption", OriginalLength: 1024} + factory := PublicationFactory{ + PackageDocument: PackageDocument{ + metadata: EPUBMetadata{global: map[string][]MetadataItem{ + VocabularyMeta + "cover": {{value: "chapter"}}, + }}, + Manifest: []Item{{ + ID: "chapter", + Href: url.MustURLFromString("OPS/text/../chapter.xhtml"), + Properties: []string{ + VocabularyItem + "cover-image", + VocabularyItem + "scripted", + VocabularyItem + "svg", + "https://example.org/custom-property", + }, + }}, + Spine: Spine{itemrefs: []ItemRef{{ + idref: "chapter", linear: true, + properties: []string{VocabularyItemref + "page-spread-left", "unknown", VocabularyRendition + "page-spread-right"}, + }}}, + }, + EncryptionData: map[string]manifest.Encryption{"OPS/chapter.xhtml": encryption}, + } + publication := factory.Create() + require.Len(t, publication.ReadingOrder, 1) + link := publication.ReadingOrder[0] + assert.Equal(t, manifest.Strings{"cover"}, link.Rels) + assert.Equal(t, manifest.Properties{ + "contains": []string{"js", "svg"}, + "https://example.org/custom-property": true, + "page": "right", + "encrypted": encryption.ToMap(), + }, link.Properties) + // Encryption matching normalizes the lookup, not the published href. + assert.Equal(t, "OPS/text/../chapter.xhtml", link.Href.String()) +} + +func TestPublicationFactoryNoLinkProperties(t *testing.T) { + for _, encryption := range []map[string]manifest.Encryption{nil, {}} { + factory := benchmarkPublicationFactory(32, false) + factory.EncryptionData = encryption + publication := factory.Create() + for _, links := range []manifest.LinkList{publication.ReadingOrder, publication.Resources} { + for _, link := range links { + assert.Nil(t, link.Properties) + } + } + } +} + +func TestPublicationFactoryRepeatedMetadata(t *testing.T) { + for _, fixture := range []string{"titles-epub3", "title-multiple-subtitles", "collections-epub3", "collections-epub2"} { + t.Run(fixture, func(t *testing.T) { + document, resourceErr := fetcher.ReadResourceAsXML(t.Context(), fetcher.NewFileResource( + manifest.Link{}, "testdata/package/"+fixture+".opf")) + require.Nil(t, resourceErr) + packageDocument, err := ParsePackageDocument(document, url.MustURLFromString("OPS/content.opf")) + require.NoError(t, err) + factory := PublicationFactory{PackageDocument: *packageDocument, FallbackTitle: "Fallback title"} + first := factory.Create() + assert.Equal(t, first, factory.Create()) + assert.Equal(t, first, factory.Create()) + }) + } +} diff --git a/pkg/parser/epub/metadata.go b/pkg/parser/epub/metadata.go index 6ba59a33..01713a9a 100644 --- a/pkg/parser/epub/metadata.go +++ b/pkg/parser/epub/metadata.go @@ -2,6 +2,7 @@ package epub import ( "errors" + "maps" "sort" "strings" "time" @@ -353,9 +354,7 @@ func (m MetadataParser) computeMetaItem(expr MetadataItem, metas map[string][]Me } children := make(map[string][]MetadataItem) - for k, v := range expr.children { - children[k] = v - } + maps.Copy(children, expr.children) for _, child := range newChildren { children[child.property] = append(children[child.property], child) } @@ -468,10 +467,14 @@ type PubMetadataAdapter struct { _subjects []manifest.Subject _allContributors map[string][]manifest.Contributor _layout manifest.Layout - _otherMetadata map[string]interface{} + _otherMetadata map[string]any } func (m PubMetadataAdapter) Metadata() manifest.Metadata { + // The public getters use value receivers. Seed their shared derived data on + // this local adapter once so each getter does not rebuild the same values. + m.seedTitleData() + m.seedBelongsToData() identifier, altIdentifiers := m.Identifiers() metadata := manifest.Metadata{ Identifier: identifier, @@ -597,6 +600,11 @@ func (m *PubMetadataAdapter) seedTitleData() { } if mainTitle != nil { m._localizedTitle = (*mainTitle).value + if mainTitle.typ == "subtitle" { + // With no main title, the first subtitle can also serve as the + // publication title. Keep their mutable translations independent. + m._localizedTitle.Translations = maps.Clone(m._localizedTitle.Translations) + } } if m._localizedTitle.String() == "" { m._localizedTitle = manifest.NewLocalizedStringFromString(m.fallbackTitle) @@ -1068,7 +1076,7 @@ func (m *PubMetadataAdapter) MediaOverlay() *manifest.MediaOverlay { } } -func (m *PubMetadataAdapter) OtherMetadata() map[string]interface{} { +func (m *PubMetadataAdapter) OtherMetadata() map[string]any { if m._otherMetadata == nil { usedProperties := map[string]struct{}{ VocabularyMeta + "cover": {}, // EPUB 2 cover meta @@ -1103,7 +1111,7 @@ func (m *PubMetadataAdapter) OtherMetadata() map[string]interface{} { VocabularySchema + "accessibilityHazard": {}, } - m._otherMetadata = make(map[string]interface{}) + m._otherMetadata = make(map[string]any) for k, v := range m.items { if _, ok := usedProperties[k]; ok { continue @@ -1131,7 +1139,7 @@ type MetadataItem struct { refines string id string children map[string][]MetadataItem - otherMetadata map[string]interface{} + otherMetadata map[string]any } func (m MetadataItem) ToSubject() (*manifest.Subject, error) { @@ -1251,11 +1259,11 @@ func (m MetadataItem) ToCollection() (string, *manifest.Contributor, error) { return m.ToContributor() } -func (m MetadataItem) ToMap() interface{} { +func (m MetadataItem) ToMap() any { if len(m.children) == 0 { return m.value } else { - cm := make(map[string]interface{}) + cm := make(map[string]any) for _, child := range m.children { for _, item := range child { cm[item.property] = item.ToMap() @@ -1331,9 +1339,7 @@ func (m MetadataItem) LocalizedString() manifest.LocalizedString { values := make(map[string]string) values[m.lang] = m.value if as := m.AlternateScript(); as != nil { - for k, v := range as { - values[k] = v - } + maps.Copy(values, as) } return manifest.NewLocalizedStringFromStrings(values) } diff --git a/pkg/parser/epub/metadata_performance_test.go b/pkg/parser/epub/metadata_performance_test.go new file mode 100644 index 00000000..d4bfcaca --- /dev/null +++ b/pkg/parser/epub/metadata_performance_test.go @@ -0,0 +1,42 @@ +package epub + +import ( + "testing" + + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A subtitle can double as the publication title when no main title exists, +// but editing either output field must not change the other one. +func TestMetadataTitleSubtitleTranslationsIndependent(t *testing.T) { + adapter := PubMetadataAdapter{ + metadataAdapter: metadataAdapter{ + items: map[string][]MetadataItem{ + VocabularyDCTerms + "title": {{ + property: VocabularyDCTerms + "title", + value: "Subtitle only", + children: map[string][]MetadataItem{ + VocabularyMeta + "title-type": {{value: "subtitle"}}, + }, + }}, + }, + }, + } + metadata := adapter.Metadata() + require.NotNil(t, metadata.LocalizedSubtitle) + assert.Equal(t, "Subtitle only", metadata.LocalizedTitle.String()) + assert.Equal(t, "Subtitle only", metadata.LocalizedSubtitle.String()) + + metadata.LocalizedTitle.SetDefaultTranslation("Edited title") + assert.Equal(t, "Subtitle only", metadata.LocalizedSubtitle.String()) + metadata.LocalizedSubtitle.SetTranslation("fr", "Sous-titre") + assert.NotContains(t, metadata.LocalizedTitle.Translations, "fr") + + // The optimization seeds only the value receiver's local copy. Later + // conversions must continue to produce fresh, independent metadata. + fresh := adapter.Metadata() + assert.Equal(t, manifest.NewLocalizedStringFromString("Subtitle only"), fresh.LocalizedTitle) + assert.Equal(t, manifest.NewLocalizedStringFromString("Subtitle only"), *fresh.LocalizedSubtitle) +} diff --git a/pkg/parser/epub/metadata_test.go b/pkg/parser/epub/metadata_test.go index 9999c04f..3ba2fb73 100644 --- a/pkg/parser/epub/metadata_test.go +++ b/pkg/parser/epub/metadata_test.go @@ -554,17 +554,17 @@ func TestMetadataOtherMetadata(t *testing.T) { m3, err := loadMetadata(t.Context(), "meta-others") require.NoError(t, err) - assert.Equal(t, m3.OtherMetadata, map[string]interface{}{ - VocabularyDCTerms + "source": []interface{}{ + assert.Equal(t, m3.OtherMetadata, map[string]any{ + VocabularyDCTerms + "source": []any{ "Feedbooks", - map[string]interface{}{"@value": "Web", "http://my.url/#scheme": "http"}, + map[string]any{"@value": "Web", "http://my.url/#scheme": "http"}, "Internet", }, "http://idpf.org/epub/vocab/package/meta/#Sigil%20version": "1.9.20", "http://www.idpf.org/2007/opf#version": "3.0", - "http://my.url/#property0": map[string]interface{}{ + "http://my.url/#property0": map[string]any{ "@value": "refines0", - "http://my.url/#property1": map[string]interface{}{ + "http://my.url/#property1": map[string]any{ "@value": "refines1", "http://my.url/#property2": "refines2", "http://my.url/#property3": "refines3", diff --git a/pkg/parser/epub/navigation_bench_test.go b/pkg/parser/epub/navigation_bench_test.go new file mode 100644 index 00000000..e036214b --- /dev/null +++ b/pkg/parser/epub/navigation_bench_test.go @@ -0,0 +1,40 @@ +package epub + +import ( + "testing" + + "github.com/antchfx/xmlquery" + "github.com/readium/go-toolkit/pkg/fetcher" + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/readium/go-toolkit/pkg/util/url" +) + +// BenchmarkNavigationDOM measures extraction from an already parsed DOM so the +// cost of navigation traversal is visible independently of XML decoding and I/O. +func BenchmarkNavigationDOM(b *testing.B) { + for _, format := range []struct { + name, directory, extension string + parse func(*xmlquery.Node, url.URL) map[string]manifest.LinkList + }{ + {"NCX", "ncx/ncx-", ".ncx", ParseNCX}, + {"NavDoc", "navdoc/nav-", ".xhtml", ParseNavDoc}, + } { + for _, fixture := range []string{"complex", "children", "titles"} { + b.Run(format.name+"/"+fixture, func(b *testing.B) { + document, err := fetcher.ReadResourceAsXML(b.Context(), fetcher.NewFileResource( + manifest.Link{}, "testdata/"+format.directory+fixture+format.extension)) + if err != nil { + b.Fatal(err) + } + base := url.MustURLFromString("OPS/navigation" + format.extension) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if len(format.parse(document, base)["toc"]) == 0 { + b.Fatal("fixture produced no navigation") + } + } + }) + } + } +} diff --git a/pkg/parser/epub/parser.go b/pkg/parser/epub/parser.go index 3049a550..4fc9287f 100644 --- a/pkg/parser/epub/parser.go +++ b/pkg/parser/epub/parser.go @@ -2,6 +2,7 @@ package epub import ( "context" + "slices" "github.com/antchfx/xmlquery" "github.com/pkg/errors" @@ -41,14 +42,14 @@ func (p Parser) Parse(ctx context.Context, asset asset.PublicationAsset, f fetch // Detect DRM - opfXmlDocument, errx := fetcher.ReadResourceAsXML(ctx, f.Get(ctx, manifest.Link{Href: manifest.NewHREF(opfPath)})) + opfData, errx := f.Get(ctx, manifest.Link{Href: manifest.NewHREF(opfPath)}).Read(ctx, 0, 0) if errx != nil { return nil, errx } - packageDocument, err := ParsePackageDocument(opfXmlDocument, opfPath) + packageDocument, err := parsePackageDocumentData(opfData, opfPath) if err != nil { - return nil, errors.Wrap(err, "invalid OPF file") + return nil, err } // Detect the container-level DRM scheme. This is done unconditionally, @@ -141,11 +142,8 @@ func parseNavigationData(ctx context.Context, packageDocument PackageDocument, f } else { var navItem *Item for _, v := range packageDocument.Manifest { - for _, st := range v.Properties { - if st == VocabularyItem+"nav" { - navItem = &v - break - } + if slices.Contains(v.Properties, VocabularyItem+"nav") { + navItem = &v } if navItem != nil { break diff --git a/pkg/parser/epub/parser_bench_test.go b/pkg/parser/epub/parser_bench_test.go index bb235fb0..ae52362e 100644 --- a/pkg/parser/epub/parser_bench_test.go +++ b/pkg/parser/epub/parser_bench_test.go @@ -1,43 +1,73 @@ package epub import ( - "context" + "io/fs" "os" "path/filepath" + "strings" "testing" "github.com/readium/go-toolkit/pkg/fetcher" - "github.com/stretchr/testify/require" ) +// BenchmarkParser measures a fresh archive open and EPUB parse for each book in +// /publications. Set EPUB_BENCH_DIR to use another local corpus. Each operation +// closes its fetcher; archive-entry and publication caches cannot leak across +// iterations. This does not read spine content or invoke lazy services. func BenchmarkParser(b *testing.B) { - pubsDir := "../../../publications" - files, err := os.ReadDir(pubsDir) - if err != nil { - b.Skip("publications directory not found") - } - - for _, file := range files { - if file.IsDir() || filepath.Ext(file.Name()) != ".epub" { - continue + files, pubsDir := epubBenchmarkFiles(b) + for _, path := range files { + name, err := filepath.Rel(pubsDir, path) + if err != nil { + b.Fatal(err) } - - b.Run(file.Name(), func(b *testing.B) { - path := filepath.Join(pubsDir, file.Name()) - - parser := NewParser(nil) - asset := fakeEPUBAsset{name: file.Name()} - for i := 0; i < b.N; i++ { - // We recreate fetcher in each iteration because in real world we parse from fresh fetcher. - f, err := fetcher.NewArchiveFetcherFromPath(context.Background(), path) - require.NoError(b, err) - func() { - defer f.Close() - builder, err := parser.Parse(context.Background(), asset, f) - require.NoError(b, err) - require.NotNil(b, builder) - }() + b.Run(filepath.ToSlash(name), func(b *testing.B) { + p := NewParser(nil) + a := fakeEPUBAsset{name: filepath.Base(path)} + ctx := b.Context() + b.ReportAllocs() + for b.Loop() { + f, err := fetcher.NewArchiveFetcherFromPath(ctx, path) + if err != nil { + b.Fatal(err) + } + builder, err := p.Parse(ctx, a, f) + f.Close() + if err != nil { + b.Fatal(err) + } + if builder == nil { + b.Fatal("EPUB was not recognized") + } } }) } } + +func epubBenchmarkFiles(b *testing.B) ([]string, string) { + b.Helper() + pubsDir := os.Getenv("EPUB_BENCH_DIR") + if pubsDir == "" { + pubsDir = "../../../publications" + } + var files []string + err := filepath.WalkDir(pubsDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.IsDir() && strings.EqualFold(filepath.Ext(path), ".epub") { + files = append(files, path) + } + return nil + }) + if os.IsNotExist(err) && os.Getenv("EPUB_BENCH_DIR") == "" { + b.Skip("local EPUB corpus not found; set EPUB_BENCH_DIR") + } + if err != nil { + b.Fatal(err) + } + if len(files) == 0 { + b.Skip("no EPUBs in corpus") + } + return files, pubsDir +} diff --git a/pkg/parser/epub/parser_navdoc.go b/pkg/parser/epub/parser_navdoc.go index 18bfe6b6..d8d69981 100644 --- a/pkg/parser/epub/parser_navdoc.go +++ b/pkg/parser/epub/parser_navdoc.go @@ -11,8 +11,6 @@ import ( var ( xpNavBody = mustCompileNS("//html:body") xpNavNav = mustCompileNS("//html:nav") - xpNavOL = mustCompileNS("html:ol") - xpNavLI = mustCompileNS("html:li") ) func ParseNavDoc(document *xmlquery.Node, filePath url.URL) map[string]manifest.LinkList { @@ -60,7 +58,7 @@ func parseNavElement(nav *xmlquery.Node, filePath url.URL, prefixMap map[string] types = append(types, resolveProperty(prop, prefixMap, DefaultVocabType)) } - links := parseOlElement(xmlquery.QuerySelector(nav, xpNavOL), filePath) + links := parseOlElement(firstNavigationChild(nav, NamespaceXHTML, "ol"), filePath) if len(links) > 0 && len(types) > 0 { return types, links } @@ -71,9 +69,11 @@ func parseOlElement(ol *xmlquery.Node, filePath url.URL) manifest.LinkList { if ol == nil { return nil } - lis := xmlquery.QuerySelectorAll(ol, xpNavLI) - links := make(manifest.LinkList, 0, len(lis)) - for _, li := range lis { + links := make(manifest.LinkList, 0, countNavigationChildren(ol, NamespaceXHTML, "li")) + for li := ol.FirstChild; li != nil; li = li.NextSibling { + if !matchesNavigationElement(li, NamespaceXHTML, "li") { + continue + } l := parseLiElement(li, filePath) if l != nil { links = append(links, *l) @@ -86,7 +86,12 @@ func parseLiElement(li *xmlquery.Node, filePath url.URL) (link *manifest.Link) { if li == nil { return nil } - first := li.SelectElement("*") // should be , , or
    + first := li.FirstChild // should be , , or
      , in any namespace + for first != nil && first.Type != xmlquery.ElementNode && first.Type != xmlquery.ProcessingInstruction { + first = first.NextSibling + } + // xmlquery's XPath navigator treats processing instructions as elements; + // retain that behavior when replacing its previous "*" selection. if first == nil { return nil } @@ -102,7 +107,7 @@ func parseLiElement(li *xmlquery.Node, filePath url.URL) (link *manifest.Link) { } } - children := parseOlElement(xmlquery.QuerySelector(li, xpNavOL), filePath) + children := parseOlElement(firstNavigationChild(li, NamespaceXHTML, "ol"), filePath) // A nil href here is the lazy stand-in for the old "#" default, whose // String() is "" — so a missing or empty-resolved href drops the entry // (unless it has children), exactly as before. @@ -118,3 +123,29 @@ func parseLiElement(li *xmlquery.Node, filePath url.URL) (link *manifest.Link) { Children: children, } } + +// These helpers implement the direct-child steps used repeatedly by navigation +// parsing, without constructing an XPath iterator for each short selection. +func matchesNavigationElement(node *xmlquery.Node, namespace, name string) bool { + return (node.Type == xmlquery.ElementNode || node.Type == xmlquery.ProcessingInstruction) && + node.NamespaceURI == namespace && node.Data == name +} + +func firstNavigationChild(parent *xmlquery.Node, namespace, name string) *xmlquery.Node { + for child := parent.FirstChild; child != nil; child = child.NextSibling { + if matchesNavigationElement(child, namespace, name) { + return child + } + } + return nil +} + +func countNavigationChildren(parent *xmlquery.Node, namespace, name string) int { + count := 0 + for child := parent.FirstChild; child != nil; child = child.NextSibling { + if matchesNavigationElement(child, namespace, name) { + count++ + } + } + return count +} diff --git a/pkg/parser/epub/parser_navdoc_test.go b/pkg/parser/epub/parser_navdoc_test.go index a84b7ee8..edab9841 100644 --- a/pkg/parser/epub/parser_navdoc_test.go +++ b/pkg/parser/epub/parser_navdoc_test.go @@ -2,8 +2,10 @@ package epub import ( "context" + "strings" "testing" + "github.com/antchfx/xmlquery" "github.com/readium/go-toolkit/pkg/fetcher" "github.com/readium/go-toolkit/pkg/manifest" "github.com/readium/go-toolkit/pkg/util/url" @@ -123,3 +125,46 @@ func TestNavDocParserPageList(t *testing.T) { {Title: "2", Href: manifest.MustNewHREFFromString("OEBPS/xhtml/chapter1.xhtml#page2", false)}, }, n["page-list"]) } + +func TestNavDocParserDirectChildrenAndNamespaces(t *testing.T) { + const source = ` + + Ignored foreign list + Ignored nested list + + Ignored foreign item + Ignored nested item + Leading textForeign anchor + Part + Ignored foreign child list + Ignored nested child list + Child + + Empty children + First child has no linkIgnored second anchor + Ignored after processing instruction + Text before elementLast + + Ignored second direct list + +` + for _, prefix := range []string{"prefixed", "default"} { + t.Run(prefix, func(t *testing.T) { + input := source + if prefix == "default" { + input = strings.NewReplacer("xmlns:h=", "xmlns=", " + + Foreign point + Nested point + + Foreign label + Foreign text + Nested text + First matching titleSecond text + Later title + + + + Child + + Ignored after empty text + Ignored after empty src + + + Foreign page + Nested page + 1 + +` + for _, prefix := range []string{"prefixed", "default"} { + t.Run(prefix, func(t *testing.T) { + input := source + if prefix == "default" { + input = strings.NewReplacer("xmlns:n=", "xmlns=", "= depth { + prefixes[attr.Value] = opfXMLPrefix{depth: depth} + } + } else if attr.Name.Space == "xmlns" { + prefixes[attr.Value] = opfXMLPrefix{name: attr.Name.Local, depth: depth} + } + } + // encoding/xml permits undeclared element prefixes; xmlquery does not. + if token.Name.Space != "" { + if _, declared := prefixes[token.Name.Space]; !declared { + return nil, false + } + } + if depth == 1 { + if root != nil || token.Name.Space != NamespaceOPF || token.Name.Local != "package" { + return nil, false + } + node := opfXMLNode(token, prefixes, nil) + root = &node + xmlquery.AddChild(document, root) + vocabulary = make(map[string]string, len(PackageReservedPrefixes)) + maps.Copy(vocabulary, PackageReservedPrefixes) + maps.Copy(vocabulary, parsePrefixes(root.SelectAttr("prefix"))) + continue + } + if token.Name.Space == NamespaceOPF && token.Name.Local == "metadata" { + // Metadata uses descendant queries, including its language lookup. + // Keep the full parser for metadata outside the one direct section. + if depth != 2 || seenMetadata { + return nil, false + } + seenMetadata = true + } + if depth == 2 { + section = "" + if token.Name.Space == NamespaceOPF { + switch token.Name.Local { + case "metadata": + section = "metadata" + case "manifest": + if seenManifest { + return nil, false + } + seenManifest, section = true, "manifest" + case "spine": + if seenSpine { + return nil, false + } + seenSpine, section = true, "spine" + } + } + if section != "" { + node := opfXMLNode(token, prefixes, nil) + xmlquery.AddChild(root, &node) + if section == "metadata" { + metadataParent = &node + } + } + continue + } + if metadataParent != nil { + node := opfXMLNode(token, prefixes, nil) + xmlquery.AddChild(metadataParent, &node) + metadataParent = &node + } else if depth == 3 && token.Name.Space == NamespaceOPF { + // ParseItem and ParseItemRef retain values, not the temporary node + // or attributes, so the small attribute buffer is safe to reuse. + switch { + case section == "manifest" && token.Name.Local == "item": + node := opfXMLNode(token, prefixes, attrBuffer[:0]) + if item := ParseItem(&node, filePath, vocabulary); item != nil { + items = append(items, item) + } + case section == "spine" && token.Name.Local == "itemref": + node := opfXMLNode(token, prefixes, attrBuffer[:0]) + if itemref := ParseItemRef(&node, vocabulary); itemref != nil { + itemrefs = append(itemrefs, itemref) + } + } + } + case xml.EndElement: + depth-- + if metadataParent != nil { + if depth == 1 { + metadataParent = nil + } else { + metadataParent = metadataParent.Parent + } + } + case xml.CharData: + if metadataParent != nil { + // Metadata consumes InnerText, for which text and CDATA are + // equivalent. Retain all whitespace and nested text in order. + xmlquery.AddChild(metadataParent, &xmlquery.Node{Type: xmlquery.TextNode, Data: string(token)}) + } + case xml.ProcInst: + if token.Target != "xml" || depth != 0 || root != nil { + return nil, false + } + case xml.Directive: + if depth != 0 || root != nil { + return nil, false + } + // A leading DOCTYPE is ignored by the reference parser as well; + // custom entities are not expanded by either decoder. + } + } + if root == nil || !seenMetadata || !seenManifest || !seenSpine { + return nil, false + } + // Reuse metadata refinements, localization, package attributes and spine + // settings from the reference parser, using only the small retained tree. + result, err := ParsePackageDocument(document, filePath) + if err != nil { + return nil, false + } + if len(items) > 0 { + result.Manifest = make([]Item, len(items)) + for i, item := range items { + result.Manifest[i] = *item + } + } + if len(itemrefs) > 0 { + result.Spine.itemrefs = make([]ItemRef, len(itemrefs)) + for i, itemref := range itemrefs { + result.Spine.itemrefs[i] = *itemref + } + } + return result, true +} diff --git a/pkg/parser/epub/parser_packagedoc_stream_test.go b/pkg/parser/epub/parser_packagedoc_stream_test.go new file mode 100644 index 00000000..903dd28e --- /dev/null +++ b/pkg/parser/epub/parser_packagedoc_stream_test.go @@ -0,0 +1,291 @@ +package epub + +import ( + "bytes" + "encoding/xml" + "math" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/antchfx/xmlquery" + "github.com/pkg/errors" + "github.com/readium/go-toolkit/pkg/fetcher" + "github.com/readium/go-toolkit/pkg/util/url" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/encoding/unicode" +) + +// parsePackageDocumentDOM preserves the complete previous Parser.Parse OPF path, +// including the distinction between XML resource errors and OPF model errors. +func parsePackageDocumentDOM(data []byte, path url.URL) (*PackageDocument, error) { + document, err := xmlquery.ParseWithOptions(bytes.NewReader(data), xmlquery.ParserOptions{ + Decoder: &xmlquery.DecoderOptions{Strict: true, Entity: xml.HTMLEntity}, + }) + if err != nil { + return nil, fetcher.Other(err) + } + model, err := ParsePackageDocument(document, path) + if err != nil { + return nil, errors.Wrap(err, "invalid OPF file") + } + return model, nil +} + +func assertPackageDocumentDataMatchesDOM(t *testing.T, data []byte) { + t.Helper() + path := url.MustURLFromString("OPS/package.opf") + want, wantErr := parsePackageDocumentDOM(data, path) + got, gotErr := parsePackageDocumentData(data, path) + if wantErr == nil { + require.NoError(t, gotErr) + assert.Equal(t, want, got) + return + } + require.Error(t, gotErr) + assert.IsType(t, wantErr, gotErr) + assert.EqualError(t, gotErr, wantErr.Error()) + assert.Equal(t, want, got) + if wantResourceErr, ok := wantErr.(*fetcher.ResourceError); ok { + gotResourceErr, ok := gotErr.(*fetcher.ResourceError) + require.True(t, ok) + assert.Equal(t, wantResourceErr.Code, gotResourceErr.Code) + assert.IsType(t, wantResourceErr.Cause, gotResourceErr.Cause) + assert.EqualError(t, gotResourceErr.Cause, wantResourceErr.Cause.Error()) + } +} + +func TestPackageDocumentDataMatchesFixtures(t *testing.T) { + fixtures, err := filepath.Glob("testdata/package/*.opf") + require.NoError(t, err) + require.NotEmpty(t, fixtures) + for _, fixture := range fixtures { + t.Run(filepath.Base(fixture), func(t *testing.T) { + data, err := os.ReadFile(fixture) + require.NoError(t, err) + assertPackageDocumentDataMatchesDOM(t, data) + }) + } +} + +const streamTestPackageStart = `` +const streamTestMetadata = `urn:test:bookA booken` +const streamTestManifest = `` +const streamTestSpine = `` + +func streamTestPackage(metadata, manifest, spine string) string { + return streamTestPackageStart + metadata + manifest + spine + `` +} + +func streamTestPrefixedPackage(data string) string { + return strings.NewReplacer( + `xmlns="http://www.idpf.org/2007/opf"`, `xmlns:opf="http://www.idpf.org/2007/opf"`, + "` + standard}, + {"EPUB2", strings.Replace(standard, `version="3.0"`, `version="2.0"`, 1)}, + {"defaultVersion", strings.Replace(standard, ` version="3.0"`, "", 1)}, + {"invalidVersion", strings.Replace(standard, `version="3.0"`, `version="invalid"`, 1)}, + {"invalidVersionBeforeItemProperties", strings.Replace(strings.Replace(standard, `version="3.0"`, `version="invalid"`, 1), `id="chapter"`, `id="chapter" properties=":"`, 1)}, + {"missingMetadataBeforeItemrefProperties", streamTestPackage("", streamTestManifest, ``)}, + {"trailingXMLErrorBeforeProperties", strings.Replace(standard, `id="chapter"`, `id="chapter" properties=":"`, 1) + `Français日本語`, streamTestManifest, streamTestSpine)}, + {"languageInLaterMetadata", streamTestPackage(`A bookfr`, streamTestManifest, streamTestSpine)}, + {"namespacedRootAttributes", strings.Replace(standard, `version="3.0"`, `xmlns:f="urn:foreign" f:version="2.0" version="3.0"`, 1)}, + {"sameNamespacePrefixAfterDefault", strings.Replace(standard, `version="3.0"`, `xmlns:v="http://www.idpf.org/2007/opf" v:version="2.0" version="3.0"`, 1)}, + {"sameNamespacePrefixBeforeDefault", strings.Replace(standard, `A multilingual 日本語 book`, streamTestManifest, streamTestSpine)}, + {"CDATA", streamTestPackage(` & 日本語]]>`, streamTestManifest, streamTestSpine)}, + {"comments", `` + streamTestPackage(``+strings.Replace(streamTestMetadata, "A book", "A book", 1)+``, streamTestManifest, streamTestSpine) + ``}, + {"processingInstructions", `` + streamTestPackage(``+strings.Replace(streamTestMetadata, "A book", "A book", 1), ``+streamTestManifest, streamTestSpine) + ``}, + {"HTMLEntities", strings.Replace(standard, "A book", `A book © 日  `, 1)}, + {"rootPrefixDeclarations", strings.Replace(standard, `version="3.0"`, `prefix="custom: https://example.org/vocab schema: https://schema.org/" version="3.0"`, 1)}, + {"metadataRefinements", streamTestPackage(`MainPrincipalMain, The12.5`, streamTestManifest, streamTestSpine)}, + {"metadataLinks", streamTestPackage(`Book`, streamTestManifest, streamTestSpine)}, + {"multipleMetadata", streamTestPackage(streamTestMetadata+`Second title`, streamTestManifest, streamTestSpine)}, + {"nestedMetadata", streamTestPackage(``+streamTestMetadata+``, streamTestManifest, streamTestSpine)}, + {"nestedAndDirectMetadata", streamTestPackage(`First nested`+streamTestMetadata, streamTestManifest, streamTestSpine)}, + {"foreignMetadata", streamTestPackage(`Foreign`+streamTestMetadata, streamTestManifest, streamTestSpine)}, + {"metadataNamespaceRedeclaration", streamTestPackage(`Foreign DC namespace`, streamTestManifest, streamTestSpine)}, + {"metadataPrefixRebound", streamTestPackage(`FrançaisAuthor`, streamTestManifest, streamTestSpine)}, + {"namespacePrefixSurvivesMetadata", streamTestPackage(strings.Replace(streamTestMetadata, ``, ``, 1), ``, streamTestSpine)}, + {"emptyMetadata", streamTestPackage(``, streamTestManifest, streamTestSpine)}, + {"emptyManifestAndSpine", streamTestPackage(streamTestMetadata, ``, ``)}, + {"duplicateManifest", streamTestPackage(streamTestMetadata, streamTestManifest+``, streamTestSpine)}, + {"duplicateSpine", streamTestPackage(streamTestMetadata, streamTestManifest, streamTestSpine+``)}, + {"sectionsReordered", streamTestPackage(streamTestSpine, streamTestManifest, streamTestMetadata)}, + {"foreignManifestItems", streamTestPackage(streamTestMetadata, ``, streamTestSpine)}, + {"foreignSpineItems", streamTestPackage(streamTestMetadata, streamTestManifest, ``)}, + {"nestedItemMarkup", streamTestPackage(streamTestMetadata, ``, streamTestSpine)}, + {"missingItemHref", streamTestPackage(streamTestMetadata, ``, streamTestSpine)}, + {"missingItemrefID", streamTestPackage(streamTestMetadata, streamTestManifest, ``)}, + {"itemPropertiesAndURIs", streamTestPackage(streamTestMetadata, ``, ``)}, + {"duplicateRootAttributes", strings.Replace(standard, `version="3.0"`, `version="3.0" version="2.0"`, 1)}, + {"duplicateItemAttributes", strings.Replace(standard, `id="chapter"`, `id="chapter" id="later"`, 1)}, + {"duplicateSpineAttributes", strings.Replace(standard, `page-progression-direction="rtl"`, `page-progression-direction="rtl" page-progression-direction="ltr"`, 1)}, + {"prefixedItemAttributes", strings.Replace(standard, `id="chapter"`, `xmlns:f="urn:foreign" f:id="foreign" id="chapter"`, 1)}, + {"undeclaredAttributeNamespace", strings.Replace(standard, `id="chapter"`, `undeclared:id="foreign" id="chapter"`, 1)}, + {"undeclaredElementNamespace", streamTestPackage(streamTestMetadata, ``, streamTestSpine)}, + {"undeclaredRootNamespace", strings.ReplaceAll(streamTestPrefixedPackage(standard), ` xmlns:opf="http://www.idpf.org/2007/opf"`, "")}, + {"missingOPFNamespace", strings.Replace(standard, ` xmlns="http://www.idpf.org/2007/opf"`, "", 1)}, + {"rootWrapped", `` + standard + ``}, + {"missingMetadata", streamTestPackage("", streamTestManifest, streamTestSpine)}, + {"missingManifest", streamTestPackage(streamTestMetadata, "", streamTestSpine)}, + {"missingSpine", streamTestPackage(streamTestMetadata, streamTestManifest, "")}, + {"emptyInput", ""}, + {"truncatedRoot", strings.TrimSuffix(standard, ``)}, + {"mismatchedClosingTag", strings.Replace(standard, ``, ``, 1)}, + {"unclosedAttribute", strings.Replace(standard, `href="text/chapter.xhtml"`, `href="text/chapter.xhtml`, 1)}, + {"unknownEntity", strings.Replace(standard, "A book", "A &undefined; book", 1)}, + {"invalidUTF8", strings.Replace(standard, "A book", "A \xff book", 1)}, + {"trailingText", standard + "trailing text"}, + {"trailingUnclosedTag", standard + ``}, + {"multiplePackages", standard + strings.Replace(standard, "A book", "Another book", 1)}, + {"leadingForeignRoot", `` + standard}, + {"doctype", `` + standard}, + {"doctypeInternalSubset", `]>` + standard}, + } { + t.Run(test.name, func(t *testing.T) { + assertPackageDocumentDataMatchesDOM(t, []byte(test.source)) + }) + } +} + +func TestPackageDocumentDataEncodedXMLMatchesDOM(t *testing.T) { + standard := streamTestPackage(streamTestMetadata, streamTestManifest, streamTestSpine) + latin1 := []byte(`` + strings.Replace(standard, "A book", "Un caf\xe9", 1)) + utf16 := []byte(`` + strings.Replace(standard, "A book", "日本語", 1)) + littleEndian, err := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes(utf16) + require.NoError(t, err) + bigEndian, err := unicode.UTF16(unicode.BigEndian, unicode.UseBOM).NewEncoder().Bytes(utf16) + require.NoError(t, err) + for _, test := range []struct { + name string + data []byte + }{ + {"ISO-8859-1", latin1}, + {"UTF-16LE", littleEndian}, + {"UTF-16BE", bigEndian}, + {"unknownEncoding", []byte(`` + standard)}, + } { + t.Run(test.name, func(t *testing.T) { + assertPackageDocumentDataMatchesDOM(t, test.data) + }) + } +} + +func TestPackageDocumentStreamStandardFastPath(t *testing.T) { + standard := streamTestPackage(streamTestMetadata, streamTestManifest, streamTestSpine) + for _, test := range []struct{ name, source string }{ + {"defaultNamespace", standard}, + {"prefixedNamespace", streamTestPrefixedPackage(standard)}, + {"EPUB2", strings.Replace(standard, `version="3.0"`, `version="2.0"`, 1)}, + {"emptySections", streamTestPackage(``, ``, ``)}, + } { + t.Run(test.name, func(t *testing.T) { + path := url.MustURLFromString("OPS/package.opf") + want, err := parsePackageDocumentDOM([]byte(test.source), path) + require.NoError(t, err) + got, fast := parsePackageDocumentStream([]byte(test.source), path) + require.True(t, fast, "ordinary OPF should use the streaming path") + assert.Equal(t, want, got) + }) + } +} + +func FuzzPackageDocumentDataMatchesDOM(f *testing.F) { + standard := streamTestPackage(streamTestMetadata, streamTestManifest, streamTestSpine) + for _, seed := range []string{ + standard, + streamTestPrefixedPackage(standard), + streamTestPackage(` & 日本語]]>Nested title`, streamTestManifest, streamTestSpine), + streamTestPackage(`A book`, streamTestManifest, streamTestSpine), + strings.Replace(standard, ` 128<<10 { + t.Skip("limit the work per fuzz input") + } + // Bound nesting and element count before entering the reference DOM + // parser, whose recursive traversals were not designed as fuzz targets. + decoder := xml.NewDecoder(bytes.NewReader(data)) + decoder.Entity = xml.HTMLEntity + depth, elements := 0, 0 + for { + token, err := decoder.Token() + if err != nil { + break + } + switch token.(type) { + case xml.StartElement: + depth++ + elements++ + if depth > 128 || elements > 4096 { + t.Skip("limit DOM depth and size") + } + case xml.EndElement: + depth-- + } + } + + path := url.MustURLFromString("OPS/package.opf") + var want *PackageDocument + var wantErr error + baselineReturned := false + func() { + // Existing DOM parser panics are outside this optimization's scope. + // Keep recovery restricted to the baseline so new parser panics fail. + defer func() { _ = recover() }() + want, wantErr = parsePackageDocumentDOM(data, path) + baselineReturned = true + }() + if !baselineReturned { + t.Skip("reference parser panics on this input") + } + + got, gotErr := parsePackageDocumentData(data, path) + if wantErr != nil { + require.Error(t, gotErr) + assert.IsType(t, wantErr, gotErr) + assert.EqualError(t, gotErr, wantErr.Error()) + assert.Equal(t, want, got) + if wantResourceErr, ok := wantErr.(*fetcher.ResourceError); ok { + gotResourceErr, ok := gotErr.(*fetcher.ResourceError) + require.True(t, ok) + assert.Equal(t, wantResourceErr.Code, gotResourceErr.Code) + assert.IsType(t, wantResourceErr.Cause, gotResourceErr.Cause) + assert.EqualError(t, gotResourceErr.Cause, wantResourceErr.Cause.Error()) + } + return + } + require.NoError(t, gotErr) + require.NotNil(t, got) + // strconv.ParseFloat accepts NaN as a version in the reference parser; + // matching NaNs are equivalent even though DeepEqual treats them unequal. + if math.IsNaN(want.EPUBVersion) && math.IsNaN(got.EPUBVersion) { + want.EPUBVersion, got.EPUBVersion = 0, 0 + } + assert.Equal(t, want, got) + }) +} diff --git a/pkg/parser/epub/parser_smil.go b/pkg/parser/epub/parser_smil.go index 439cefe0..0e4da697 100644 --- a/pkg/parser/epub/parser_smil.go +++ b/pkg/parser/epub/parser_smil.go @@ -10,23 +10,25 @@ import ( "github.com/readium/go-toolkit/pkg/util/url" ) -var ( - xpSMILRoot = mustCompileNS("/smil:smil | /smil2:smil") - xpSMILBody = mustCompileNS("smil:body | smil2:body") - xpSMILParOrSeq = mustCompileNS("smil:par | smil:seq | smil2:par | smil2:seq") - xpSMILTextChild = mustCompileNS("smil:text | smil2:text") - xpSMILAudio = mustCompileNS("smil:audio | smil2:audio") -) +// The previous XPath unions select the SMIL namespace before SMIL 2, even +// when a SMIL 2 sibling appears earlier in the document. Keep that precedence +// while avoiding the XPath union's costly node hashing and deduplication. +func firstSMILChild(parent *xmlquery.Node, name string) *xmlquery.Node { + if child := firstNavigationChild(parent, NamespaceSMIL, name); child != nil { + return child + } + return firstNavigationChild(parent, NamespaceSMIL2, name) +} func ParseSMILDocument(document *xmlquery.Node, filePath url.URL) (*guidednavigation.GuidedNavigationDocument, error) { - smil := xmlquery.QuerySelector(document, xpSMILRoot) + smil := firstSMILChild(document, "smil") if smil == nil { return nil, errors.New("SMIL root element not found") } // Ignore the , we don't need it with the current implementation - body := xmlquery.QuerySelector(smil, xpSMILBody) + body := firstSMILChild(smil, "body") if body == nil { return nil, errors.New("SMIL body not found") } @@ -41,56 +43,78 @@ func ParseSMILDocument(document *xmlquery.Node, filePath url.URL) (*guidednaviga } func ParseSMILSeq(seq *xmlquery.Node, filePath url.URL) ([]guidednavigation.GuidedNavigationObject, error) { - childElements := xmlquery.QuerySelectorAll(seq, xpSMILParOrSeq) - if len(childElements) == 0 && seq.Data == "body" { + childCount := 0 + for child := seq.FirstChild; child != nil; child = child.NextSibling { + if child.Data == "par" || child.Data == "seq" { + if matchesNavigationElement(child, NamespaceSMIL, child.Data) || + matchesNavigationElement(child, NamespaceSMIL2, child.Data) { + childCount++ + } + } + } + if childCount == 0 && seq.Data == "body" { return nil, errors.New("SMIL body is empty") } - objects := make([]guidednavigation.GuidedNavigationObject, 0, len(childElements)) - for _, el := range childElements { - if el.Data == "par" { - // - o, err := ParseSMILPar(el, filePath) - if err != nil { - return nil, errors.Wrap(err, "failed parsing SMIL par") - } - objects = append(objects, *o) - } else { - // - textrefAttr := SelectNodeAttrNs(el, NamespaceOPS, "textref") - if textrefAttr == "" { - return nil, errors.New("SMIL seq has no textref") - } - u, err := url.URLFromString(textrefAttr) - if err != nil { - return nil, errors.Wrap(err, "failed parsing SMIL seq textref") - } - o := &guidednavigation.GuidedNavigationObject{ - TextRef: filePath.Resolve(u), - } - - // epub:type - pp := parseProperties(SelectNodeAttrNs(el, NamespaceOPS, "type")) - if len(pp) > 0 { - o.Role = make([]guidednavigation.GuidedNavigationRole, 0, len(pp)) - for _, prop := range pp { - p := converter.ConvertEPUBRole(prop) - if p == "" { - continue - } - o.Role = append(o.Role, p) + objects := make([]guidednavigation.GuidedNavigationObject, 0, childCount) + // Match the original union's branch order: SMIL par, SMIL seq, SMIL 2 + // par, then SMIL 2 seq. Within each branch, retain sibling order. + for _, namespace := range [...]string{NamespaceSMIL, NamespaceSMIL2} { + for _, name := range [...]string{"par", "seq"} { + for child := seq.FirstChild; child != nil; child = child.NextSibling { + if !matchesNavigationElement(child, namespace, name) { + continue + } + o, err := parseSMILSequenceChild(child, filePath) + if err != nil { + return nil, err } + objects = append(objects, *o) } + } + } + return objects, nil +} + +func parseSMILSequenceChild(el *xmlquery.Node, filePath url.URL) (*guidednavigation.GuidedNavigationObject, error) { + if el.Data == "par" { + o, err := ParseSMILPar(el, filePath) + if err != nil { + return nil, errors.Wrap(err, "failed parsing SMIL par") + } + return o, nil + } - // child elements - children, err := ParseSMILSeq(el, filePath) - if err != nil { - return nil, errors.Wrap(err, "failed parsing SMIL seq children") + textrefAttr := SelectNodeAttrNs(el, NamespaceOPS, "textref") + if textrefAttr == "" { + return nil, errors.New("SMIL seq has no textref") + } + u, err := url.URLFromString(textrefAttr) + if err != nil { + return nil, errors.Wrap(err, "failed parsing SMIL seq textref") + } + o := &guidednavigation.GuidedNavigationObject{ + TextRef: filePath.Resolve(u), + } + + // epub:type + pp := parseProperties(SelectNodeAttrNs(el, NamespaceOPS, "type")) + if len(pp) > 0 { + o.Role = make([]guidednavigation.GuidedNavigationRole, 0, len(pp)) + for _, prop := range pp { + p := converter.ConvertEPUBRole(prop) + if p == "" { + continue } - o.Children = children - objects = append(objects, *o) + o.Role = append(o.Role, p) } } - return objects, nil + + children, err := ParseSMILSeq(el, filePath) + if err != nil { + return nil, errors.Wrap(err, "failed parsing SMIL seq children") + } + o.Children = children + return o, nil } func secondsToDuration(seconds *float64) *time.Duration { @@ -102,7 +126,7 @@ func secondsToDuration(seconds *float64) *time.Duration { } func ParseSMILPar(par *xmlquery.Node, filePath url.URL) (*guidednavigation.GuidedNavigationObject, error) { - text := xmlquery.QuerySelector(par, xpSMILTextChild) + text := firstSMILChild(par, "text") if text == nil { return nil, errors.New("SMIL par has no text element") } @@ -119,7 +143,7 @@ func ParseSMILPar(par *xmlquery.Node, filePath url.URL) (*guidednavigation.Guide } // Audio is optional - if audio := xmlquery.QuerySelector(par, xpSMILAudio); audio != nil { + if audio := firstSMILChild(par, "audio"); audio != nil { audioAttr := audio.SelectAttr("src") if audioAttr == "" { return nil, errors.New("SMIL par audio element has empty src attribute") diff --git a/pkg/parser/epub/parser_smil_performance_test.go b/pkg/parser/epub/parser_smil_performance_test.go new file mode 100644 index 00000000..6927f927 --- /dev/null +++ b/pkg/parser/epub/parser_smil_performance_test.go @@ -0,0 +1,148 @@ +package epub + +import ( + "fmt" + "strings" + "testing" + + "github.com/antchfx/xmlquery" + "github.com/readium/go-toolkit/pkg/util/url" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSMILMixedNamespaceSelectionAndOrder(t *testing.T) { + document, err := xmlquery.Parse(strings.NewReader(` + + + + + + + + + + + + + + par + + + + `)) + require.NoError(t, err) + result, err := ParseSMILDocument(document, url.MustURLFromString("OPS/overlay.smil")) + require.NoError(t, err) + require.Len(t, result.Guided, 6) + + // The pinned XPath union selected each namespace/name branch in order, + // keeping sibling order within a branch. Preserve that existing behavior. + var refs []string + for _, object := range result.Guided { + refs = append(refs, object.TextRef.String()) + } + assert.Equal(t, []string{ + "OPS/first.xhtml", "OPS/second.xhtml", "OPS/sequence.xhtml", + "OPS/old-first.xhtml", "OPS/old-second.xhtml", "OPS/old-sequence.xhtml", + }, refs) + assert.Nil(t, result.Guided[0].AudioRef) + assert.NotNil(t, result.Guided[2].Children) + assert.Empty(t, result.Guided[2].Children) + assert.Equal(t, "OPS/first.mp3", result.Guided[3].AudioRef.String()) + assert.Equal(t, "OPS/second.mp3", result.Guided[4].AudioRef.String()) + require.Len(t, result.Guided[5].Children, 1) + assert.Equal(t, "OPS/old-child.xhtml", result.Guided[5].Children[0].TextRef.String()) +} + +func TestSMILRequiredElements(t *testing.T) { + for _, namespace := range []string{NamespaceSMIL, NamespaceSMIL2} { + t.Run(namespace, func(t *testing.T) { + for _, tc := range []struct { + name string + xml string + want string + }{ + {"foreign root", ``, "SMIL root element not found"}, + {"nested root", ``, "SMIL root element not found"}, + {"missing body", ``, "SMIL body not found"}, + {"empty body", ``, "failed parsing SMIL body: SMIL body is empty"}, + {"missing text", ``, "failed parsing SMIL body: failed parsing SMIL par: SMIL par has no text element"}, + {"empty first text", ``, "failed parsing SMIL body: failed parsing SMIL par: SMIL par text element has empty src attribute"}, + {"empty first audio", ``, "failed parsing SMIL body: failed parsing SMIL par: SMIL par audio element has empty src attribute"}, + {"foreign textref", ``, "failed parsing SMIL body: SMIL seq has no textref"}, + {"invalid seq textref", ``, `failed parsing SMIL body: failed parsing SMIL seq textref: parse "%": invalid URL escape "%"`}, + } { + t.Run(tc.name, func(t *testing.T) { + // Place namespace declarations on a document wrapper, then pass + // that wrapper as the query root, as supported by xmlquery. + xml := `` + tc.xml + `` + document, err := xmlquery.Parse(strings.NewReader(xml)) + require.NoError(t, err) + root := document.FirstChild + for root != nil && root.Type != xmlquery.ElementNode { + root = root.NextSibling + } + require.NotNil(t, root) + result, err := ParseSMILDocument(root, url.MustURLFromString("OPS/overlay.smil")) + require.EqualError(t, err, tc.want) + assert.Nil(t, result) + }) + } + }) + } +} + +func TestSMILNamespacePrecedenceDoesNotSkipInvalidElement(t *testing.T) { + for _, tc := range []struct { + name string + xml string + want string + }{ + {"body", ``, "failed parsing SMIL body: SMIL body is empty"}, + {"text", ``, "failed parsing SMIL body: failed parsing SMIL par: SMIL par text element has empty src attribute"}, + {"audio", ``, "failed parsing SMIL body: failed parsing SMIL par: SMIL par audio element has empty src attribute"}, + } { + t.Run(tc.name, func(t *testing.T) { + document, err := xmlquery.Parse(strings.NewReader(`` + tc.xml + ``)) + require.NoError(t, err) + result, err := ParseSMILDocument(document, url.MustURLFromString("OPS/overlay.smil")) + require.EqualError(t, err, tc.want) + assert.Nil(t, result) + }) + } +} + +// Keep DOM parsing outside the timed loop so sibling counts expose conversion +// cost rather than XML tokenization or archive I/O. +func BenchmarkSMILPreparedDOM(b *testing.B) { + for _, size := range []int{32, 256, 2048} { + b.Run(fmt.Sprintf("pars=%d", size), func(b *testing.B) { + var xml strings.Builder + xml.WriteString(``) + for i := range size { + fmt.Fprintf(&xml, "\n", i, i, i+1) + } + xml.WriteString("\n") + document, err := xmlquery.Parse(strings.NewReader(xml.String())) + if err != nil { + b.Fatal(err) + } + path := url.MustURLFromString("OPS/overlay.smil") + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + result, err := ParseSMILDocument(document, path) + if err != nil { + b.Fatal(err) + } + if len(result.Guided) != size { + b.Fatalf("expected %d objects, got %d", size, len(result.Guided)) + } + } + }) + } +} diff --git a/pkg/parser/epub/parser_stages_bench_test.go b/pkg/parser/epub/parser_stages_bench_test.go new file mode 100644 index 00000000..a47486b0 --- /dev/null +++ b/pkg/parser/epub/parser_stages_bench_test.go @@ -0,0 +1,155 @@ +package epub + +import ( + "bytes" + "encoding/xml" + "path/filepath" + "testing" + + "github.com/antchfx/xmlquery" + "github.com/readium/go-toolkit/pkg/fetcher" + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/readium/go-toolkit/pkg/protection" +) + +// BenchmarkEPUBStages separates archive opening, OPF XML decoding, OPF model +// construction, navigation loading/parsing, DRM detection, and manifest creation. +// Except ArchiveOpen, stages reuse an open archive; OPFModel and ManifestFactory +// also reuse prepared input structures. These diagnose CPU/allocation costs, not +// cold I/O latency. Use BenchmarkParser for complete fresh-archive measurements. +// OPFDecode compares full XML-to-model decoding with the retained DOM reference. +func BenchmarkEPUBStages(b *testing.B) { + files, dir := epubBenchmarkFiles(b) + for _, path := range files { + name, err := filepath.Rel(dir, path) + if err != nil { + b.Fatal(err) + } + b.Run(filepath.ToSlash(name), func(b *testing.B) { + ctx := b.Context() + f, err := fetcher.NewArchiveFetcherFromPath(ctx, path) + if err != nil { + b.Fatal(err) + } + b.Cleanup(f.Close) + opfPath, err := GetRootFilePath(ctx, f) + if err != nil { + b.Fatal(err) + } + res := f.Get(ctx, manifest.Link{Href: manifest.NewHREF(opfPath)}) + raw, rerr := res.Read(ctx, 0, 0) + res.Close() + if rerr != nil { + b.Fatal(rerr) + } + decode := func() (*xmlquery.Node, error) { + return xmlquery.ParseWithOptions(bytes.NewReader(raw), xmlquery.ParserOptions{ + Decoder: &xmlquery.DecoderOptions{Strict: true, Entity: xml.HTMLEntity}, + }) + } + doc, err := decode() + if err != nil { + b.Fatal(err) + } + pkg, err := ParsePackageDocument(doc, opfPath) + if err != nil { + b.Fatal(err) + } + scheme, encDoc, err := protection.IdentifyEPUBProtection(ctx, f) + if err != nil { + b.Fatal(err) + } + encryption, err := parseEncryptionData(ctx, f, scheme.URI(), encDoc) + if err != nil { + b.Fatal(err) + } + factory := PublicationFactory{ + FallbackTitle: filepath.Base(path), PackageDocument: *pkg, + NavigationData: parseNavigationData(ctx, *pkg, f), + EncryptionData: encryption, DisplayOptions: parseDisplayOptions(ctx, f), + } + b.Run("ArchiveOpen", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + archive, err := fetcher.NewArchiveFetcherFromPath(ctx, path) + if err != nil { + b.Fatal(err) + } + archive.Close() + } + }) + b.Run("OPFXML", func(b *testing.B) { + b.ReportAllocs() + b.SetBytes(int64(len(raw))) + for b.Loop() { + if _, err := decode(); err != nil { + b.Fatal(err) + } + } + }) + b.Run("OPFModel", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if _, err := ParsePackageDocument(doc, opfPath); err != nil { + b.Fatal(err) + } + } + }) + b.Run("OPFDecode", func(b *testing.B) { + for _, implementation := range []string{"DOM", "Streaming"} { + b.Run(implementation, func(b *testing.B) { + _, streamed := parsePackageDocumentStream(raw, opfPath) + b.ReportAllocs() + b.SetBytes(int64(len(raw))) + for b.Loop() { + var parsed *PackageDocument + var err error + if implementation == "DOM" { + var node *xmlquery.Node + node, err = decode() + if err == nil { + parsed, err = ParsePackageDocument(node, opfPath) + } + } else { + parsed, err = parsePackageDocumentData(raw, opfPath) + } + if err != nil { + b.Fatal(err) + } + if parsed == nil || len(parsed.Manifest) != len(pkg.Manifest) || len(parsed.Spine.itemrefs) != len(pkg.Spine.itemrefs) { + b.Fatal("OPF decoder returned different item counts") + } + } + if implementation == "Streaming" && streamed { + b.ReportMetric(1, "streamed/op") + } + }) + } + }) + b.Run("Navigation", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + parseNavigationData(ctx, *pkg, f) + } + }) + b.Run("Protection", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + scheme, doc, err := protection.IdentifyEPUBProtection(ctx, f) + if err != nil { + b.Fatal(err) + } + if _, err := parseEncryptionData(ctx, f, scheme.URI(), doc); err != nil { + b.Fatal(err) + } + } + }) + b.Run("ManifestFactory", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + factory.Create() + } + }) + }) + } +} diff --git a/pkg/parser/epub/parser_test.go b/pkg/parser/epub/parser_test.go index cffb419f..dda61feb 100644 --- a/pkg/parser/epub/parser_test.go +++ b/pkg/parser/epub/parser_test.go @@ -18,7 +18,7 @@ import ( // themselves and pass it to [Parser.Parse] directly. type fakeEPUBAsset struct{ name string } -func (a fakeEPUBAsset) Name() string { return a.name } +func (a fakeEPUBAsset) Name() string { return a.name } func (a fakeEPUBAsset) MediaType(context.Context) mediatype.MediaType { return mediatype.EPUB } func (a fakeEPUBAsset) CreateFetcher(context.Context, asset.Dependencies, string) (fetcher.Fetcher, error) { return nil, errors.New("unused in tests") @@ -67,7 +67,7 @@ func TestParserEndToEnd(t *testing.T) { encryptedCount := 0 for _, link := range append(m.ReadingOrder, m.Resources...) { - enc, ok := link.Properties["encrypted"].(map[string]interface{}) + enc, ok := link.Properties["encrypted"].(map[string]any) if !ok { continue } diff --git a/pkg/parser/epub/positions_service.go b/pkg/parser/epub/positions_service.go index 985c43c0..f0762681 100644 --- a/pkg/parser/epub/positions_service.go +++ b/pkg/parser/epub/positions_service.go @@ -107,7 +107,7 @@ func (s *PositionsService) createReflowable(ctx context.Context, link manifest.L positionCount := s.reflowableStrategy.PositionCount(resource) positions := make([]manifest.Locator, positionCount) - for p := uint(0); p < positionCount; p++ { + for p := range positionCount { positions[p] = s.createLocator( link, float64(p)/float64(positionCount), @@ -185,7 +185,7 @@ func (l ArchiveEntryLength) PositionCount(resource fetcher.Resource) uint { var length uint64 props := resource.Properties() if p := props.Get("https://readium.org/webpub-manifest/properties#archive"); p != nil { - if pm, ok := p.(map[string]interface{}); ok { + if pm, ok := p.(map[string]any); ok { if el, ok := pm["entryLength"].(uint64); ok { length = el } diff --git a/pkg/parser/utils.go b/pkg/parser/utils.go index 624a39e3..2af3d968 100644 --- a/pkg/parser/utils.go +++ b/pkg/parser/utils.go @@ -11,7 +11,7 @@ import ( func hrefCommonFirstComponent(links manifest.LinkList) string { latest := "" for _, link := range links { - normalized := strings.SplitN(link.URL(nil, nil).Path(), "/", 2)[0] + normalized, _, _ := strings.Cut(link.URL(nil, nil).Path(), "/") if latest != "" { if latest != normalized { latest = "" // No distinct prefix diff --git a/pkg/parser/webpub/parser.go b/pkg/parser/webpub/parser.go index e5e98bab..47bdb6a7 100644 --- a/pkg/parser/webpub/parser.go +++ b/pkg/parser/webpub/parser.go @@ -38,7 +38,7 @@ func (p WebPubParser) Parse(ctx context.Context, a asset.PublicationAsset, f fet isPackage := !mediaType.IsRwpm() lFetcher := f - var manifestJSON map[string]interface{} + var manifestJSON map[string]any if isPackage { res := lFetcher.Get(ctx, manifest.Link{Href: manifest.MustNewHREFFromString("manifest.json", false)}) mjr, rerr := fetcher.ReadResourceAsJSON(ctx, res) diff --git a/pkg/parser/webpub/parser_test.go b/pkg/parser/webpub/parser_test.go index 64d329e2..e18aa47e 100644 --- a/pkg/parser/webpub/parser_test.go +++ b/pkg/parser/webpub/parser_test.go @@ -70,7 +70,7 @@ func TestParseBareAudiobookManifest(t *testing.T) { // The manifest itself is reachable at its own name. data, err = readPublicationResource(t, p, "manifest.json") require.NoError(t, err) - var mjson map[string]interface{} + var mjson map[string]any require.NoError(t, json.Unmarshal(data, &mjson)) // A missing sibling fails. diff --git a/pkg/pub/publication.go b/pkg/pub/publication.go index c3d6d2be..19696c77 100644 --- a/pkg/pub/publication.go +++ b/pkg/pub/publication.go @@ -64,7 +64,7 @@ func (p Publication) PositionsFromManifest(ctx context.Context) []manifest.Locat if !ok { return []manifest.Locator{} } - positions, ok := rawPositions.([]map[string]interface{}) + positions, ok := rawPositions.([]map[string]any) if !ok { return []manifest.Locator{} } diff --git a/pkg/pub/service_positions.go b/pkg/pub/service_positions.go index d42c8691..af450bef 100644 --- a/pkg/pub/service_positions.go +++ b/pkg/pub/service_positions.go @@ -38,7 +38,7 @@ func GetForPositionsService(ctx context.Context, service PositionsService, link return fetcher.NewBytesResource(PositionsLink, func() []byte { positions := service.Positions(ctx) - bin, _ := json.Marshal(map[string]interface{}{ + bin, _ := json.Marshal(map[string]any{ "total": len(positions), "positions": positions, }) diff --git a/pkg/streamer/epub_bench_test.go b/pkg/streamer/epub_bench_test.go new file mode 100644 index 00000000..a9049efb --- /dev/null +++ b/pkg/streamer/epub_bench_test.go @@ -0,0 +1,99 @@ +package streamer + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/readium/go-toolkit/pkg/asset" + "github.com/readium/go-toolkit/pkg/mediatype" + "github.com/readium/go-toolkit/pkg/util/url" +) + +// BenchmarkEPUBOpen measures the complete local opening path, including archive +// creation, EPUB parsing, service construction, and publication cleanup. Set +// EPUB_BENCH_DIR to use another corpus. The default corpus is optional and is not +// needed to run tests on a checkout without local publications. +// +// KnownMediaType avoids media-type detection. SniffedMediaType creates a fresh asset +// on every iteration so the asset's media-type cache cannot hide sniffing costs. +// For .epub files, detection trusts the extension without inspecting ZIP content. +// WithPositions uses a known media type and also computes the complete default +// position list once for each freshly opened publication. This uses archive-entry +// sizes for reflowable content; it does not parse spine HTML or media overlays. +// +// Select individual modes and books with -bench, for example: +// +// go test ./pkg/streamer -run '^$' -bench 'BenchmarkEPUBOpen/KnownMediaType/moby-dick' -benchmem +func BenchmarkEPUBOpen(b *testing.B) { + files, pubsDir := epubBenchmarkFiles(b) + for _, mode := range []string{"KnownMediaType", "SniffedMediaType", "WithPositions"} { + b.Run(mode, func(b *testing.B) { + for _, path := range files { + name, err := filepath.Rel(pubsDir, path) + if err != nil { + b.Fatal(err) + } + b.Run(filepath.ToSlash(name), func(b *testing.B) { + uri, err := url.FromFilepath(path) + if err != nil { + b.Fatal(err) + } + s := New(Config{}) + ctx := b.Context() + var positionCount int + b.ReportAllocs() + for b.Loop() { + var a *asset.FileAsset + if mode == "SniffedMediaType" { + a = asset.File(uri) + } else { + a = asset.FileWithMediaType(uri, &mediatype.EPUB) + } + publication, err := s.Open(ctx, a, "") + if err != nil { + b.Fatal(err) + } + if mode == "WithPositions" { + positionCount += len(publication.Positions(ctx)) + } + publication.Close() + } + if mode == "WithPositions" { + b.ReportMetric(float64(positionCount)/float64(b.N), "positions/op") + } + }) + } + }) + } +} + +func epubBenchmarkFiles(b *testing.B) ([]string, string) { + b.Helper() + pubsDir := os.Getenv("EPUB_BENCH_DIR") + if pubsDir == "" { + pubsDir = "../../publications" + } + var files []string + err := filepath.WalkDir(pubsDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.IsDir() && strings.EqualFold(filepath.Ext(path), ".epub") { + files = append(files, path) + } + return nil + }) + if os.IsNotExist(err) && os.Getenv("EPUB_BENCH_DIR") == "" { + b.Skip("local EPUB corpus not found; set EPUB_BENCH_DIR") + } + if err != nil { + b.Fatal(err) + } + if len(files) == 0 { + b.Skip("no EPUBs in corpus") + } + return files, pubsDir +} diff --git a/pkg/streamer/epub_content_bench_test.go b/pkg/streamer/epub_content_bench_test.go new file mode 100644 index 00000000..29e85d94 --- /dev/null +++ b/pkg/streamer/epub_content_bench_test.go @@ -0,0 +1,118 @@ +package streamer + +import ( + "context" + "path/filepath" + "testing" + + "github.com/readium/go-toolkit/pkg/asset" + "github.com/readium/go-toolkit/pkg/fetcher" + "github.com/readium/go-toolkit/pkg/guidednavigation/converter" + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/readium/go-toolkit/pkg/mediatype" + "github.com/readium/go-toolkit/pkg/parser/epub" + "github.com/readium/go-toolkit/pkg/pub" + "github.com/readium/go-toolkit/pkg/util/url" +) + +// BenchmarkEPUBContent measures the lazy content work excluded by +// BenchmarkEPUBOpen. For each book it selects the largest declared HTML or SMIL +// resource by uncompressed length, outside the timer. Each iteration fetches, +// decompresses, parses, and converts that resource to guided navigation again. +// It bypasses publication service caches and closes the resource each time. +// +// The archive remains open between iterations, matching repeated content access +// within one publication. Publication opening, resource selection, and closing +// are excluded. EPUB_BENCH_DIR selects an optional local corpus as for +// BenchmarkEPUBOpen. Books without a matching resource are skipped. +func BenchmarkEPUBContent(b *testing.B) { + files, pubsDir := epubBenchmarkFiles(b) + for _, kind := range []string{"HTML", "SMIL"} { + b.Run(kind, func(b *testing.B) { + for _, path := range files { + name, err := filepath.Rel(pubsDir, path) + if err != nil { + b.Fatal(err) + } + b.Run(filepath.ToSlash(name), func(b *testing.B) { + uri, err := url.FromFilepath(path) + if err != nil { + b.Fatal(err) + } + ctx := b.Context() + s := New(Config{}) + publication, err := s.Open(ctx, asset.FileWithMediaType(uri, &mediatype.EPUB), "") + if err != nil { + b.Fatal(err) + } + b.Cleanup(publication.Close) + link, length := largestEPUBBenchmarkResource(b, ctx, publication, kind) + locator := manifest.Locator{ + Href: link.URL(nil, nil), + MediaType: *link.MediaType, + Title: link.Title, + } + b.ReportAllocs() + b.SetBytes(length) + for b.Loop() { + resource := publication.Fetcher.Get(ctx, link) + if kind == "HTML" { + document, err := converter.Do(ctx, resource, locator) + resource.Close() + if err != nil { + b.Fatalf("%s: %v", link.Href.String(), err) + } + if document == nil { + b.Fatal("HTML conversion returned no document") + } + } else { + node, rerr := fetcher.ReadResourceAsXML(ctx, resource) + resource.Close() + if rerr != nil { + b.Fatalf("%s: %v", link.Href.String(), rerr) + } + document, err := epub.ParseSMILDocument(node, locator.Href) + if err != nil { + b.Fatalf("%s: %v", link.Href.String(), err) + } + if document == nil { + b.Fatal("SMIL conversion returned no document") + } + } + } + b.ReportMetric(float64(length), "input-bytes/op") + }) + } + }) + } +} + +func largestEPUBBenchmarkResource(b *testing.B, ctx context.Context, publication *pub.Publication, kind string) (manifest.Link, int64) { + b.Helper() + var largest manifest.Link + var maxLength int64 = -1 + for _, links := range []manifest.LinkList{publication.Manifest.ReadingOrder, publication.Manifest.Resources} { + for _, link := range links { + if link.MediaType == nil { + continue + } + matches := kind == "HTML" && link.MediaType.IsHTML() || kind == "SMIL" && link.MediaType.Equal(&mediatype.SMIL) + if !matches { + continue + } + resource := publication.Fetcher.Get(ctx, link) + length, err := resource.Length(ctx) + resource.Close() + if err != nil { + b.Fatalf("reading %s length: %v", link.Href.String(), err) + } + if length > maxLength || length == maxLength && link.Href.String() < largest.Href.String() { + largest, maxLength = link, length + } + } + } + if maxLength < 0 { + b.Skipf("publication declares no %s resources", kind) + } + return largest, maxLength +}