From 3a0659c2826c759d4f3f869122d4da548c7bd745 Mon Sep 17 00:00:00 2001 From: AnnatarHe Date: Sat, 1 Aug 2026 23:00:22 +0800 Subject: [PATCH 1/3] feat(cli): add Kindle SDR highlight extraction --- README.md | 47 +- cmd/ck-cli/main.go | 5 +- internal/commands/sdr.go | 108 +++++ internal/commands/sdr_test.go | 27 ++ internal/sdr/azw3.go | 832 ++++++++++++++++++++++++++++++++++ internal/sdr/extract.go | 242 ++++++++++ internal/sdr/krds.go | 387 ++++++++++++++++ internal/sdr/sdr_test.go | 352 ++++++++++++++ internal/sdr/text.go | 117 +++++ internal/sdr/types.go | 61 +++ 10 files changed, 2174 insertions(+), 4 deletions(-) create mode 100644 internal/commands/sdr.go create mode 100644 internal/commands/sdr_test.go create mode 100644 internal/sdr/azw3.go create mode 100644 internal/sdr/extract.go create mode 100644 internal/sdr/krds.go create mode 100644 internal/sdr/sdr_test.go create mode 100644 internal/sdr/text.go create mode 100644 internal/sdr/types.go diff --git a/README.md b/README.md index 59244c8..cd41cfc 100644 --- a/README.md +++ b/README.md @@ -19,21 +19,56 @@ cat "My Clippings.txt" | ck-cli parse > output.json # Extract unique titles cat "My Clippings.txt" | ck-cli parse | jq -r .[].title | sort -u + +# Extract highlights from a mounted Kindle +ck-cli sdr --path "/Volumes/Kindle/documents" + +# Extract one book as ClippingItem JSON +ck-cli sdr --path "/path/to/Book.sdr" --json ``` -**Options:** +**Parse options:** + - `-i, --input`: Input file path (default: stdin) - `-o, --output`: Output file path or `http` for web sync (default: stdout) **Output format:** + ```json [{ "title": "Book Title", "content": "Highlighted text", - "pageAt": "78", + "pageAt": "#78", "createdAt": "2019-03-27T19:57:26Z" }] ``` + +### Kindle `.sdr` Highlights + +Recent Kindle sidecars store annotations as positions rather than embedding the +highlighted words. The `sdr` command pairs each `.azw3r` sidecar with its sibling +AZW3/KF8 book and reconstructs the selected text locally: + +```bash +# Recursively scan a Kindle root or documents directory +ck-cli sdr --path "/Volumes/Kindle/documents" + +# Process a single sidecar directory or book +ck-cli sdr --path "/path/to/Book.sdr" +ck-cli sdr --path "/path/to/Book.azw3" --json +``` + +`--path` accepts a Kindle root/documents tree, one `.sdr` directory, or one +`.azw3`, `.azw`, or KF8-containing `.mobi` file. The default output is readable +text grouped by book. `--json` emits the same `title`, `content`, `pageAt`, and +`createdAt` schema as `parse`; printed APNX pages are preferred, with the raw +annotation position used as a fallback. + +The implementation is read-only, offline, and written natively in Go—Python and +KindleUnpack are not runtime dependencies. It supports unencrypted AZW3/KF8 books +with `.azw3r` sidecars. DRM-protected books, Mobi7, and KFX/`.yjr` are skipped as +unsupported. + ### Web Sync ```bash @@ -66,6 +101,7 @@ See [Makefile](./Makefile) for all commands. - Flexible I/O (files, stdin/stdout, web sync) - High-performance processing of large files - Direct ClippingKK web service integration +- Native Kindle `.sdr`/`.azw3r` highlight extraction - Cross-platform (macOS, Linux, Windows) ## Contributing @@ -73,4 +109,11 @@ See [Makefile](./Makefile) for all commands. See [CLAUDE.md](./CLAUDE.md) for development guidelines. ## License + [MIT](https://choosealicense.com/licenses/mit/) + +The `.sdr` implementation was informed by the published format research in +[kindle-reading-dashboard](https://github.com/zevisvei/kindle-reading-dashboard) +and the container behavior documented by +[KindleUnpack](https://github.com/kevinhendricks/KindleUnpack). No code from +either GPLv3 project is bundled or required. diff --git a/cmd/ck-cli/main.go b/cmd/ck-cli/main.go index 5de51cb..057178b 100644 --- a/cmd/ck-cli/main.go +++ b/cmd/ck-cli/main.go @@ -15,7 +15,7 @@ import ( var ( // Version is set at build time Version = "dev" - // Commit is set at build time + // Commit is set at build time Commit = "unknown" ) @@ -60,6 +60,7 @@ func main() { Commands: []*cli.Command{ commands.LoginCommand, commands.ParseCommand, + commands.SDRCommand, }, Before: func(c *cli.Context) error { // Inject global configuration context @@ -71,4 +72,4 @@ func main() { if err := app.RunContext(ctx, os.Args); err != nil { log.Fatal(err) } -} \ No newline at end of file +} diff --git a/internal/commands/sdr.go b/internal/commands/sdr.go new file mode 100644 index 0000000..5ae1b35 --- /dev/null +++ b/internal/commands/sdr.go @@ -0,0 +1,108 @@ +package commands + +import ( + "fmt" + "io" + "os" + "strings" + "unicode/utf8" + + "github.com/clippingkk/cli/internal/models" + "github.com/clippingkk/cli/internal/sdr" + "github.com/urfave/cli/v2" +) + +// SDRCommand extracts highlights from Kindle .sdr sidecars and their books. +var SDRCommand = &cli.Command{ + Name: "sdr", + Usage: "Extract highlighted text from Kindle .sdr sidecars", + Description: `Read Kindle .sdr sidecars and recover highlighted text from their +unencrypted AZW3/KF8 books. The path may be a mounted Kindle or documents tree, +a single .sdr directory, or a single AZW3/KF8 book. + +Examples: + ck-cli sdr --path /Volumes/Kindle/documents + ck-cli sdr --path "Book.sdr" --json + ck-cli sdr --path "Book.azw3" --json`, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "path", + Aliases: []string{"p"}, + Usage: "Kindle documents tree, .sdr directory, or AZW3/KF8 book", + Required: true, + }, + &cli.BoolFlag{ + Name: "json", + Usage: "Output the existing ClippingItem JSON format", + }, + }, + Action: sdrAction, +} + +func sdrAction(c *cli.Context) error { + report, err := sdr.ExtractPath(c.String("path")) + for _, warning := range report.Warnings { + fmt.Fprintf(os.Stderr, "⚠️ %s\n", warning) + } + if err != nil { + return err + } + + count := 0 + for _, book := range report.Books { + count += len(book.Highlights) + } + if c.Bool("json") { + items := make([]models.ClippingItem, 0, count) + for _, book := range report.Books { + for _, highlight := range book.Highlights { + items = append(items, models.ClippingItem{ + Title: highlight.Title, Content: highlight.Text, + PageAt: highlight.PageAt, CreatedAt: highlight.CreatedAt, + }) + } + } + if err := outputJSON(os.Stdout, items); err != nil { + return err + } + } else { + if err := renderSDRText(os.Stdout, report); err != nil { + return err + } + } + + fmt.Fprintf(os.Stderr, "📚 Extracted %d text annotations from %d books\n", count, report.Decoded) + return nil +} + +func renderSDRText(writer io.Writer, report sdr.Report) error { + for _, book := range report.Books { + if len(book.Highlights) == 0 { + continue + } + if _, err := fmt.Fprintln(writer, book.Title); err != nil { + return fmt.Errorf("write output: %w", err) + } + if _, err := fmt.Fprintln(writer, strings.Repeat("=", utf8.RuneCountInString(book.Title))); err != nil { + return fmt.Errorf("write output: %w", err) + } + if _, err := fmt.Fprintln(writer); err != nil { + return fmt.Errorf("write output: %w", err) + } + for _, highlight := range book.Highlights { + if _, err := fmt.Fprintf(writer, "[%s] %s\n", highlight.Type, highlight.Text); err != nil { + return fmt.Errorf("write output: %w", err) + } + if highlight.Note != "" { + if _, err := fmt.Fprintf(writer, " Note: %s\n", highlight.Note); err != nil { + return fmt.Errorf("write output: %w", err) + } + } + if _, err := fmt.Fprintf(writer, " Location: %s | Created: %s\n\n", + highlight.PageAt, highlight.CreatedAt.UTC().Format("2006-01-02T15:04:05Z07:00")); err != nil { + return fmt.Errorf("write output: %w", err) + } + } + } + return nil +} diff --git a/internal/commands/sdr_test.go b/internal/commands/sdr_test.go new file mode 100644 index 0000000..22a693e --- /dev/null +++ b/internal/commands/sdr_test.go @@ -0,0 +1,27 @@ +package commands + +import ( + "strings" + "testing" + "time" + + "github.com/clippingkk/cli/internal/sdr" +) + +func TestRenderSDRText(t *testing.T) { + report := sdr.Report{Books: []sdr.BookResult{{ + Title: "原则", + Highlights: []sdr.Highlight{{ + Type: sdr.AnnotationNote, Text: "selected text", Note: "remember", + PageAt: "#12", CreatedAt: time.Date(2026, 7, 1, 8, 30, 0, 0, time.UTC), + }}, + }}} + var output strings.Builder + if err := renderSDRText(&output, report); err != nil { + t.Fatalf("renderSDRText() error = %v", err) + } + want := "原则\n==\n\n[note] selected text\n Note: remember\n Location: #12 | Created: 2026-07-01T08:30:00Z\n\n" + if output.String() != want { + t.Fatalf("renderSDRText() = %q, want %q", output.String(), want) + } +} diff --git a/internal/sdr/azw3.go b/internal/sdr/azw3.go new file mode 100644 index 0000000..b2cee77 --- /dev/null +++ b/internal/sdr/azw3.go @@ -0,0 +1,832 @@ +package sdr + +import ( + "bytes" + "encoding/binary" + "fmt" + "strconv" + "strings" + "unicode/utf8" +) + +const ( + mobiCompressionNone = 1 + mobiCompressionPalmDOC = 2 + mobiCompressionHUFF = 0x4448 + missingSection = uint32(0xffffffff) +) + +type palmDatabase struct { + name string + identity string + sections [][]byte +} + +type mobiBook struct { + db *palmDatabase + start int + title string + compression uint16 + textRecords int + version uint32 + huffOffset uint32 + huffCount uint32 + fdst uint32 + fdstCount uint32 + fragment uint32 + skeleton uint32 + trailerFlags uint16 +} + +type indexTag struct { + tag byte + valuesPerEntry byte + mask byte + endFlag byte +} + +type indexEntry struct { + text string + tags map[byte][]uint64 +} + +type skeletonEntry struct { + fragmentCount int + position int + length int +} + +type fragmentEntry struct { + insertPosition int + position int + length int +} + +// AssembleBook reads an unencrypted KF8 book and returns its title and +// reconstructed main markup flow. Annotation positions index this byte slice. +func AssembleBook(data []byte) (string, []byte, error) { + db, err := parsePalmDatabase(data) + if err != nil { + return "", nil, err + } + book, err := findKF8Book(db) + if err != nil { + return "", nil, err + } + raw, err := book.rawMarkup() + if err != nil { + return "", nil, err + } + assembled, err := book.assembleKF8(raw) + if err != nil { + return "", nil, err + } + return book.title, assembled, nil +} + +func parsePalmDatabase(data []byte) (*palmDatabase, error) { + if len(data) < 78 { + return nil, fmt.Errorf("book is too small to be a Palm database") + } + recordCount := int(binary.BigEndian.Uint16(data[76:78])) + if recordCount == 0 || recordCount > 100_000 { + return nil, fmt.Errorf("invalid Palm record count %d", recordCount) + } + tableEnd := 78 + recordCount*8 + if tableEnd > len(data) { + return nil, fmt.Errorf("truncated Palm record table") + } + offsets := make([]int, recordCount+1) + for i := 0; i < recordCount; i++ { + offset := uint64(binary.BigEndian.Uint32(data[78+i*8 : 82+i*8])) + if offset > uint64(len(data)) { + return nil, fmt.Errorf("Palm record %d offset is outside the file", i) + } + offsets[i] = int(offset) + if i > 0 && offsets[i] < offsets[i-1] { + return nil, fmt.Errorf("Palm record offsets are not ordered") + } + } + if offsets[0] < tableEnd { + return nil, fmt.Errorf("first Palm record overlaps the record table") + } + offsets[recordCount] = len(data) + sections := make([][]byte, recordCount) + for i := range sections { + sections[i] = data[offsets[i]:offsets[i+1]] + } + name := strings.TrimRight(string(data[:32]), "\x00") + identity := string(data[60:68]) + if identity != "BOOKMOBI" && identity != "TEXtREAd" { + return nil, fmt.Errorf("unsupported Palm identity %q", identity) + } + return &palmDatabase{name: name, identity: identity, sections: sections}, nil +} + +func findKF8Book(db *palmDatabase) (*mobiBook, error) { + if len(db.sections) == 0 { + return nil, fmt.Errorf("book has no records") + } + firstBook, firstErr := parseMobiHeader(db, 0) + if firstErr == nil && firstBook.version == 8 { + return firstBook, nil + } + for i, section := range db.sections { + if len(section) == 8 && string(section) == "BOUNDARY" && i+1 < len(db.sections) { + book, err := parseMobiHeader(db, i+1) + if err != nil { + return nil, fmt.Errorf("parse hybrid KF8 header: %w", err) + } + return book, nil + } + } + if firstErr != nil { + return nil, firstErr + } + return nil, fmt.Errorf("book has no KF8 content (Mobi7 is not supported)") +} + +func parseMobiHeader(db *palmDatabase, start int) (*mobiBook, error) { + if start < 0 || start >= len(db.sections) { + return nil, fmt.Errorf("MOBI header record is missing") + } + header := db.sections[start] + if len(header) < 40 || string(header[16:20]) != "MOBI" { + return nil, fmt.Errorf("record %d is not a MOBI header", start) + } + headerLength := binary.BigEndian.Uint32(header[20:24]) + if headerLength < 0x18 || uint64(headerLength)+16 > uint64(len(header)) { + return nil, fmt.Errorf("invalid MOBI header length %d", headerLength) + } + crypto := binary.BigEndian.Uint16(header[12:14]) + if crypto != 0 { + return nil, fmt.Errorf("book is DRM-encrypted") + } + book := &mobiBook{ + db: db, start: start, + compression: binary.BigEndian.Uint16(header[0:2]), + textRecords: int(binary.BigEndian.Uint16(header[8:10])), + version: binary.BigEndian.Uint32(header[36:40]), + title: db.name, + huffOffset: missingSection, fdst: missingSection, + fragment: missingSection, skeleton: missingSection, + } + if book.textRecords < 0 || start+book.textRecords >= len(db.sections) { + return nil, fmt.Errorf("MOBI text record count %d exceeds the file", book.textRecords) + } + if len(header) >= 0x5c { + offset := binary.BigEndian.Uint32(header[0x54:0x58]) + length := binary.BigEndian.Uint32(header[0x58:0x5c]) + if uint64(offset)+uint64(length) <= uint64(len(header)) && length > 0 { + book.title = decodeMobiText(header[offset:offset+length], binary.BigEndian.Uint32(header[28:32])) + } + } + if updated := updatedTitle(header, headerLength, binary.BigEndian.Uint32(header[28:32])); updated != "" { + book.title = updated + } + if strings.TrimSpace(book.title) == "" { + book.title = db.name + } + if len(header) >= 0x78 { + book.huffOffset = binary.BigEndian.Uint32(header[0x70:0x74]) + book.huffCount = binary.BigEndian.Uint32(header[0x74:0x78]) + } + if len(header) >= 0xc8 { + book.fdst = binary.BigEndian.Uint32(header[0xc0:0xc4]) + book.fdstCount = binary.BigEndian.Uint32(header[0xc4:0xc8]) + if book.fdstCount <= 1 { + book.fdst = missingSection + } + } + if len(header) >= 0x100 { + book.trailerFlags = binary.BigEndian.Uint16(header[0xf2:0xf4]) + book.fragment = binary.BigEndian.Uint32(header[0xf8:0xfc]) + book.skeleton = binary.BigEndian.Uint32(header[0xfc:0x100]) + } + return book, nil +} + +func updatedTitle(header []byte, mobiHeaderLength, codepage uint32) string { + if len(header) < 0x84 || binary.BigEndian.Uint32(header[0x80:0x84])&0x40 == 0 { + return "" + } + offset := int(mobiHeaderLength) + 16 + if offset+12 > len(header) || string(header[offset:offset+4]) != "EXTH" { + return "" + } + length := int(binary.BigEndian.Uint32(header[offset+4 : offset+8])) + count := int(binary.BigEndian.Uint32(header[offset+8 : offset+12])) + if length < 12 || offset+length > len(header) || count > 100_000 { + return "" + } + position := offset + 12 + for i := 0; i < count && position+8 <= offset+length; i++ { + kind := binary.BigEndian.Uint32(header[position : position+4]) + size := int(binary.BigEndian.Uint32(header[position+4 : position+8])) + if size < 8 || position+size > offset+length { + break + } + if kind == 503 { + return decodeMobiText(header[position+8:position+size], codepage) + } + position += size + } + return "" +} + +func decodeMobiText(data []byte, codepage uint32) string { + if codepage == 65001 || utf8.Valid(data) { + return strings.TrimSpace(strings.TrimRight(string(data), "\x00")) + } + // MOBI commonly uses Windows-1252. Decode its non-ISO control range. + cp1252 := [...]rune{0x20ac, 0x0081, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008d, 0x017d, 0x008f, + 0x0090, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0x009d, 0x017e, 0x0178} + var out strings.Builder + for _, b := range data { + switch { + case b >= 0x80 && b <= 0x9f: + out.WriteRune(cp1252[b-0x80]) + default: + out.WriteRune(rune(b)) + } + } + return strings.TrimSpace(strings.TrimRight(out.String(), "\x00")) +} + +func (b *mobiBook) absolute(relative uint32) (int, error) { + if relative == missingSection { + return -1, fmt.Errorf("required MOBI section is absent") + } + index := uint64(relative) + uint64(b.start) + if index >= uint64(len(b.db.sections)) { + return -1, fmt.Errorf("MOBI section %d is outside the file", index) + } + return int(index), nil +} + +func (b *mobiBook) rawMarkup() ([]byte, error) { + var huff *huffDecoder + if b.compression == mobiCompressionHUFF { + index, err := b.absolute(b.huffOffset) + if err != nil { + return nil, fmt.Errorf("locate HUFF table: %w", err) + } + if b.huffCount < 2 || uint64(index)+uint64(b.huffCount) > uint64(len(b.db.sections)) { + return nil, fmt.Errorf("invalid HUFF/CDIC section count %d", b.huffCount) + } + huff = &huffDecoder{} + if err := huff.loadHUFF(b.db.sections[index]); err != nil { + return nil, err + } + for i := 1; i < int(b.huffCount); i++ { + if err := huff.loadCDIC(b.db.sections[index+i]); err != nil { + return nil, err + } + } + } + + var out bytes.Buffer + for i := 1; i <= b.textRecords; i++ { + record, err := trimTrailingData(b.db.sections[b.start+i], b.trailerFlags) + if err != nil { + return nil, fmt.Errorf("trim text record %d: %w", i, err) + } + var decoded []byte + switch b.compression { + case mobiCompressionNone: + decoded = append([]byte(nil), record...) + case mobiCompressionPalmDOC: + decoded, err = decompressPalmDOC(record) + case mobiCompressionHUFF: + decoded, err = huff.unpack(record, 0) + default: + return nil, fmt.Errorf("unsupported MOBI compression 0x%x", b.compression) + } + if err != nil { + return nil, fmt.Errorf("decompress text record %d: %w", i, err) + } + out.Write(decoded) + } + return out.Bytes(), nil +} + +func trimTrailingData(data []byte, flags uint16) ([]byte, error) { + record := data + trailers := 0 + shifted := flags + for shifted > 1 { + if shifted&2 != 0 { + trailers++ + } + shifted >>= 1 + } + for i := 0; i < trailers; i++ { + size, err := trailingEntrySize(record) + if err != nil || size <= 0 || size > len(record) { + return nil, fmt.Errorf("invalid trailing entry size") + } + record = record[:len(record)-size] + } + if flags&1 != 0 { + if len(record) == 0 { + return nil, fmt.Errorf("missing multibyte trailer") + } + size := int(record[len(record)-1]&3) + 1 + if size > len(record) { + return nil, fmt.Errorf("invalid multibyte trailer size") + } + record = record[:len(record)-size] + } + return record, nil +} + +func trailingEntrySize(data []byte) (int, error) { + if len(data) == 0 { + return 0, fmt.Errorf("empty record") + } + start := len(data) - 4 + if start < 0 { + start = 0 + } + value := 0 + for _, b := range data[start:] { + if b&0x80 != 0 { + value = 0 + } + value = (value << 7) | int(b&0x7f) + } + return value, nil +} + +func decompressPalmDOC(input []byte) ([]byte, error) { + output := make([]byte, 0, len(input)*2) + for position := 0; position < len(input); { + code := input[position] + position++ + switch { + case code >= 1 && code <= 8: + count := int(code) + if position+count > len(input) { + return nil, fmt.Errorf("truncated PalmDOC literal") + } + output = append(output, input[position:position+count]...) + position += count + case code < 0x80: + output = append(output, code) + case code >= 0xc0: + output = append(output, ' ', code^0x80) + default: + if position >= len(input) { + return nil, fmt.Errorf("truncated PalmDOC back-reference") + } + pair := uint16(code)<<8 | uint16(input[position]) + position++ + distance := int((pair >> 3) & 0x7ff) + length := int(pair&7) + 3 + if distance == 0 || distance > len(output) { + return nil, fmt.Errorf("invalid PalmDOC back-reference distance %d", distance) + } + for i := 0; i < length; i++ { + output = append(output, output[len(output)-distance]) + } + } + } + return output, nil +} + +type huffCode struct { + length int + terminal bool + maxCode uint32 +} + +type huffPhrase struct { + data []byte + terminal bool + expanded []byte + busy bool +} + +type huffDecoder struct { + lookup [256]huffCode + minCode [33]uint32 + maxCode [33]uint32 + phrases []huffPhrase +} + +func (h *huffDecoder) loadHUFF(data []byte) error { + if len(data) < 24 || string(data[:8]) != "HUFF\x00\x00\x00\x18" { + return fmt.Errorf("invalid HUFF header") + } + offset1 := int(binary.BigEndian.Uint32(data[8:12])) + offset2 := int(binary.BigEndian.Uint32(data[12:16])) + if offset1 < 0 || offset1+256*4 > len(data) || offset2 < 0 || offset2+64*4 > len(data) { + return fmt.Errorf("HUFF tables are truncated") + } + for i := 0; i < 256; i++ { + value := binary.BigEndian.Uint32(data[offset1+i*4 : offset1+i*4+4]) + length := int(value & 0x1f) + if length == 0 || length > 32 { + return fmt.Errorf("invalid HUFF code length %d", length) + } + max := uint64(value >> 8) + max = ((max + 1) << (32 - length)) - 1 + h.lookup[i] = huffCode{length: length, terminal: value&0x80 != 0, maxCode: uint32(max)} + } + for length := 1; length <= 32; length++ { + min := binary.BigEndian.Uint32(data[offset2+(length-1)*8 : offset2+(length-1)*8+4]) + max := binary.BigEndian.Uint32(data[offset2+(length-1)*8+4 : offset2+length*8]) + h.minCode[length] = uint32(uint64(min) << (32 - length)) + h.maxCode[length] = uint32(((uint64(max) + 1) << (32 - length)) - 1) + } + return nil +} + +func (h *huffDecoder) loadCDIC(data []byte) error { + if len(data) < 16 || string(data[:8]) != "CDIC\x00\x00\x00\x10" { + return fmt.Errorf("invalid CDIC header") + } + phraseCount := int(binary.BigEndian.Uint32(data[8:12])) + bits := int(binary.BigEndian.Uint32(data[12:16])) + if bits < 0 || bits > 16 || phraseCount < len(h.phrases) { + return fmt.Errorf("invalid CDIC phrase table") + } + count := 1 << bits + if remaining := phraseCount - len(h.phrases); count > remaining { + count = remaining + } + if 16+count*2 > len(data) { + return fmt.Errorf("truncated CDIC offsets") + } + for i := 0; i < count; i++ { + offset := int(binary.BigEndian.Uint16(data[16+i*2 : 18+i*2])) + if 18+offset > len(data) { + return fmt.Errorf("CDIC phrase offset is outside the record") + } + lengthFlag := binary.BigEndian.Uint16(data[16+offset : 18+offset]) + length := int(lengthFlag & 0x7fff) + start := 18 + offset + if start+length > len(data) { + return fmt.Errorf("truncated CDIC phrase") + } + h.phrases = append(h.phrases, huffPhrase{ + data: append([]byte(nil), data[start:start+length]...), terminal: lengthFlag&0x8000 != 0, + }) + } + return nil +} + +func (h *huffDecoder) unpack(input []byte, depth int) ([]byte, error) { + if depth > 64 { + return nil, fmt.Errorf("HUFF phrase recursion limit exceeded") + } + padded := make([]byte, len(input)+8) + copy(padded, input) + bitsLeft := len(input) * 8 + position := 0 + remaining := 32 + window := binary.BigEndian.Uint64(padded[:8]) + var output bytes.Buffer + for { + if remaining <= 0 { + position += 4 + if position+8 > len(padded) { + break + } + window = binary.BigEndian.Uint64(padded[position : position+8]) + remaining += 32 + } + code := uint32(window >> remaining) + entry := h.lookup[code>>24] + length := entry.length + maxCode := entry.maxCode + if !entry.terminal { + for length <= 32 && code < h.minCode[length] { + length++ + } + if length > 32 { + return nil, fmt.Errorf("invalid HUFF code") + } + maxCode = h.maxCode[length] + } + remaining -= length + bitsLeft -= length + if bitsLeft < 0 { + break + } + phraseIndex := uint64(maxCode-code) >> (32 - length) + if phraseIndex >= uint64(len(h.phrases)) { + return nil, fmt.Errorf("HUFF phrase index %d is outside the dictionary", phraseIndex) + } + phrase := &h.phrases[phraseIndex] + if phrase.terminal { + output.Write(phrase.data) + continue + } + if phrase.busy { + return nil, fmt.Errorf("recursive HUFF dictionary cycle") + } + if phrase.expanded == nil { + phrase.busy = true + expanded, err := h.unpack(phrase.data, depth+1) + phrase.busy = false + if err != nil { + return nil, err + } + phrase.expanded = expanded + } + output.Write(phrase.expanded) + } + return output.Bytes(), nil +} + +func (b *mobiBook) assembleKF8(raw []byte) ([]byte, error) { + flow := raw + if b.fdst != missingSection { + index, err := b.absolute(b.fdst) + if err != nil { + return nil, fmt.Errorf("locate FDST: %w", err) + } + section := b.db.sections[index] + if len(section) < 12 || string(section[:4]) != "FDST" { + return nil, fmt.Errorf("invalid FDST record") + } + count := int(binary.BigEndian.Uint32(section[8:12])) + if count <= 0 || 12+count*8 > len(section) { + return nil, fmt.Errorf("invalid FDST flow count %d", count) + } + start := int(binary.BigEndian.Uint32(section[12:16])) + end := len(raw) + if count > 1 { + end = int(binary.BigEndian.Uint32(section[20:24])) + } + if start < 0 || end < start || end > len(raw) { + return nil, fmt.Errorf("FDST main flow range is invalid") + } + flow = raw[start:end] + } + if b.skeleton == missingSection || b.fragment == missingSection { + return append([]byte(nil), flow...), nil + } + skeletonIndex, err := b.absolute(b.skeleton) + if err != nil { + return nil, fmt.Errorf("locate skeleton index: %w", err) + } + fragmentIndex, err := b.absolute(b.fragment) + if err != nil { + return nil, fmt.Errorf("locate fragment index: %w", err) + } + skeletonRows, err := parseMobiIndex(b.db.sections, skeletonIndex) + if err != nil { + return nil, fmt.Errorf("parse skeleton index: %w", err) + } + fragmentRows, err := parseMobiIndex(b.db.sections, fragmentIndex) + if err != nil { + return nil, fmt.Errorf("parse fragment index: %w", err) + } + skeletons := make([]skeletonEntry, 0, len(skeletonRows)) + for _, row := range skeletonRows { + one := row.tags[1] + six := row.tags[6] + if len(one) < 1 || len(six) < 2 { + return nil, fmt.Errorf("skeleton index entry is missing required tags") + } + skeletons = append(skeletons, skeletonEntry{fragmentCount: int(one[0]), position: int(six[0]), length: int(six[1])}) + } + fragments := make([]fragmentEntry, 0, len(fragmentRows)) + for _, row := range fragmentRows { + position, err := strconv.ParseUint(row.text, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid fragment insert position %q", row.text) + } + six := row.tags[6] + if len(six) < 2 { + return nil, fmt.Errorf("fragment index entry is missing tag 6") + } + fragments = append(fragments, fragmentEntry{insertPosition: int(position), position: int(six[0]), length: int(six[1])}) + } + var assembled bytes.Buffer + fragmentPointer := 0 + for _, skeleton := range skeletons { + base := skeleton.position + skeleton.length + if skeleton.position < 0 || base < skeleton.position || base > len(flow) { + return nil, fmt.Errorf("skeleton range is outside the main flow") + } + part := append([]byte(nil), flow[skeleton.position:base]...) + for i := 0; i < skeleton.fragmentCount; i++ { + if fragmentPointer >= len(fragments) { + return nil, fmt.Errorf("skeleton references missing fragments") + } + fragment := fragments[fragmentPointer] + fragmentPointer++ + end := base + fragment.length + if fragment.length < 0 || end < base || end > len(flow) { + return nil, fmt.Errorf("fragment range is outside the main flow") + } + insert := fragment.insertPosition - skeleton.position + if insert < 0 || insert > len(part) { + return nil, fmt.Errorf("fragment insert position is outside its skeleton") + } + withFragment := make([]byte, 0, len(part)+fragment.length) + withFragment = append(withFragment, part[:insert]...) + withFragment = append(withFragment, flow[base:end]...) + withFragment = append(withFragment, part[insert:]...) + part = withFragment + base = end + } + assembled.Write(part) + } + if assembled.Len() == 0 { + return nil, fmt.Errorf("KF8 skeleton index produced no text") + } + return assembled.Bytes(), nil +} + +func parseMobiIndex(sections [][]byte, index int) ([]indexEntry, error) { + if index < 0 || index >= len(sections) { + return nil, fmt.Errorf("INDX record is missing") + } + main := sections[index] + headerLength, recordCount, _, err := parseINDXHeader(main) + if err != nil { + return nil, err + } + controlBytes, tags, err := parseTAGX(main, headerLength) + if err != nil { + return nil, err + } + if recordCount < 0 || index+recordCount >= len(sections) { + return nil, fmt.Errorf("INDX record count exceeds the file") + } + entries := make([]indexEntry, 0) + for record := 1; record <= recordCount; record++ { + data := sections[index+record] + _, entryCount, idxt, err := parseINDXHeader(data) + if err != nil { + return nil, err + } + if idxt < 0 || idxt+4+entryCount*2 > len(data) || string(data[idxt:idxt+4]) != "IDXT" { + return nil, fmt.Errorf("invalid IDXT table") + } + positions := make([]int, entryCount+1) + for i := 0; i < entryCount; i++ { + positions[i] = int(binary.BigEndian.Uint16(data[idxt+4+i*2 : idxt+6+i*2])) + } + positions[entryCount] = idxt + for i := 0; i < entryCount; i++ { + start, end := positions[i], positions[i+1] + if start < 0 || start >= end || end > len(data) { + return nil, fmt.Errorf("invalid INDX entry range") + } + textLength := int(data[start]) + valueStart := start + 1 + textLength + if valueStart > end { + return nil, fmt.Errorf("truncated INDX entry text") + } + tagMap, err := decodeIndexTags(data, valueStart, end, controlBytes, tags) + if err != nil { + return nil, err + } + entries = append(entries, indexEntry{text: string(data[start+1 : start+1+textLength]), tags: tagMap}) + } + } + return entries, nil +} + +func parseINDXHeader(data []byte) (headerLength, count, idxt int, err error) { + if len(data) < 56 || string(data[:4]) != "INDX" { + return 0, 0, 0, fmt.Errorf("invalid INDX header") + } + headerLength = int(binary.BigEndian.Uint32(data[4:8])) + idxt = int(binary.BigEndian.Uint32(data[20:24])) + count = int(binary.BigEndian.Uint32(data[24:28])) + if headerLength < 0 || headerLength > len(data) || count < 0 || count > 1_000_000 { + return 0, 0, 0, fmt.Errorf("invalid INDX header values") + } + return headerLength, count, idxt, nil +} + +func parseTAGX(data []byte, start int) (int, []indexTag, error) { + if start < 0 || start+12 > len(data) || string(data[start:start+4]) != "TAGX" { + return 0, nil, fmt.Errorf("missing TAGX table") + } + length := int(binary.BigEndian.Uint32(data[start+4 : start+8])) + controlBytes := int(binary.BigEndian.Uint32(data[start+8 : start+12])) + if length < 12 || start+length > len(data) || controlBytes <= 0 || controlBytes > 32 { + return 0, nil, fmt.Errorf("invalid TAGX header") + } + tags := make([]indexTag, 0, (length-12)/4) + for position := start + 12; position+4 <= start+length; position += 4 { + tags = append(tags, indexTag{data[position], data[position+1], data[position+2], data[position+3]}) + } + return controlBytes, tags, nil +} + +func decodeIndexTags(data []byte, start, end, controlByteCount int, table []indexTag) (map[byte][]uint64, error) { + if start < 0 || start+controlByteCount > end || end > len(data) { + return nil, fmt.Errorf("truncated INDX control bytes") + } + type pendingTag struct { + tag byte + count int + byteLength int + valuesPerEntry int + } + pending := make([]pendingTag, 0) + controlIndex := 0 + dataPosition := start + controlByteCount + for _, tag := range table { + if tag.endFlag == 1 { + controlIndex++ + continue + } + if controlIndex >= controlByteCount { + return nil, fmt.Errorf("TAGX control byte index overflow") + } + masked := data[start+controlIndex] & tag.mask + if masked == 0 { + continue + } + item := pendingTag{tag: tag.tag, valuesPerEntry: int(tag.valuesPerEntry)} + if masked == tag.mask { + if bitCount(tag.mask) > 1 { + value, consumed, err := variableWidth(data, dataPosition, end) + if err != nil { + return nil, err + } + dataPosition += consumed + item.byteLength = int(value) + } else { + item.count = 1 + } + } else { + value, mask := masked, tag.mask + for mask&1 == 0 { + mask >>= 1 + value >>= 1 + } + item.count = int(value) + } + pending = append(pending, item) + } + result := make(map[byte][]uint64) + for _, item := range pending { + values := make([]uint64, 0) + if item.byteLength > 0 { + limit := dataPosition + item.byteLength + if limit > end { + return nil, fmt.Errorf("INDX tag values exceed the entry") + } + for dataPosition < limit { + value, consumed, err := variableWidth(data, dataPosition, limit) + if err != nil { + return nil, err + } + dataPosition += consumed + values = append(values, value) + } + } else { + count := item.count * item.valuesPerEntry + for i := 0; i < count; i++ { + value, consumed, err := variableWidth(data, dataPosition, end) + if err != nil { + return nil, err + } + dataPosition += consumed + values = append(values, value) + } + } + result[item.tag] = values + } + return result, nil +} + +func variableWidth(data []byte, offset, limit int) (uint64, int, error) { + var value uint64 + for consumed := 0; consumed < 10; consumed++ { + position := offset + consumed + if position >= limit || position >= len(data) { + return 0, 0, fmt.Errorf("truncated variable-width integer") + } + b := data[position] + if value > (^uint64(0) >> 7) { + return 0, 0, fmt.Errorf("variable-width integer overflow") + } + value = (value << 7) | uint64(b&0x7f) + if b&0x80 != 0 { + return value, consumed + 1, nil + } + } + return 0, 0, fmt.Errorf("variable-width integer is too long") +} + +func bitCount(value byte) int { + count := 0 + for value != 0 { + count += int(value & 1) + value >>= 1 + } + return count +} diff --git a/internal/sdr/extract.go b/internal/sdr/extract.go new file mode 100644 index 0000000..9b69d05 --- /dev/null +++ b/internal/sdr/extract.go @@ -0,0 +1,242 @@ +package sdr + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +type bookCandidate struct { + bookPath string + sidecarDir string +} + +// ExtractPath discovers Kindle book/sidecar pairs beneath path and extracts +// all supported text annotations. It never writes to the source tree. +func ExtractPath(path string) (Report, error) { + absPath, err := filepath.Abs(path) + if err != nil { + return Report{}, fmt.Errorf("resolve path: %w", err) + } + info, err := os.Stat(absPath) + if err != nil { + return Report{}, fmt.Errorf("inspect %s: %w", path, err) + } + + candidates, direct, discoveryWarnings, err := discoverCandidates(absPath, info) + if err != nil { + return Report{}, err + } + report := Report{Warnings: discoveryWarnings} + for _, candidate := range candidates { + result, err := extractCandidate(candidate) + if err != nil { + if direct { + return Report{}, err + } + report.Warnings = append(report.Warnings, fmt.Sprintf("%s: %v", candidate.sidecarDir, err)) + continue + } + report.Decoded++ + report.Books = append(report.Books, result) + } + if report.Decoded == 0 { + if len(report.Warnings) > 0 { + return report, fmt.Errorf("no supported AZW3/KF8 sidecar pairs could be decoded") + } + return report, fmt.Errorf("no supported AZW3/KF8 sidecar pairs found under %s", path) + } + return report, nil +} + +func discoverCandidates(path string, info os.FileInfo) ([]bookCandidate, bool, []string, error) { + if !info.IsDir() { + if !supportedBookExtension(filepath.Ext(path)) { + return nil, true, nil, fmt.Errorf("unsupported input file %s", path) + } + sidecar := strings.TrimSuffix(path, filepath.Ext(path)) + ".sdr" + if sidecarInfo, err := os.Stat(sidecar); err != nil || !sidecarInfo.IsDir() { + return nil, true, nil, fmt.Errorf("sidecar directory %s was not found", sidecar) + } + return []bookCandidate{{bookPath: path, sidecarDir: sidecar}}, true, nil, nil + } + if strings.EqualFold(filepath.Ext(path), ".sdr") { + book, err := findSiblingBook(path) + if err != nil { + return nil, true, nil, err + } + return []bookCandidate{{bookPath: book, sidecarDir: path}}, true, nil, nil + } + + var candidates []bookCandidate + var warnings []string + err := filepath.WalkDir(path, func(current string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + warnings = append(warnings, fmt.Sprintf("%s: %v", current, walkErr)) + if entry != nil && entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if !entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".sdr") { + return nil + } + book, err := findSiblingBook(current) + if err != nil { + warnings = append(warnings, fmt.Sprintf("%s: %v", current, err)) + return filepath.SkipDir + } + candidates = append(candidates, bookCandidate{bookPath: book, sidecarDir: current}) + return filepath.SkipDir + }) + if err != nil { + return nil, false, warnings, fmt.Errorf("scan %s: %w", path, err) + } + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].bookPath < candidates[j].bookPath + }) + return candidates, false, warnings, nil +} + +func findSiblingBook(sidecarDir string) (string, error) { + base := strings.TrimSuffix(sidecarDir, filepath.Ext(sidecarDir)) + parent := filepath.Dir(sidecarDir) + entries, err := os.ReadDir(parent) + if err != nil { + return "", fmt.Errorf("read book directory: %w", err) + } + baseName := filepath.Base(base) + priority := map[string]int{".azw3": 0, ".azw": 1, ".mobi": 2} + type match struct { + path string + priority int + } + var matches []match + for _, entry := range entries { + if entry.IsDir() { + continue + } + ext := strings.ToLower(filepath.Ext(entry.Name())) + order, ok := priority[ext] + if !ok || !strings.EqualFold(strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())), baseName) { + continue + } + matches = append(matches, match{path: filepath.Join(parent, entry.Name()), priority: order}) + } + if len(matches) == 0 { + return "", fmt.Errorf("no sibling AZW3/KF8 book found (KFX and DRM are unsupported)") + } + sort.Slice(matches, func(i, j int) bool { + if matches[i].priority != matches[j].priority { + return matches[i].priority < matches[j].priority + } + return matches[i].path < matches[j].path + }) + return matches[0].path, nil +} + +func supportedBookExtension(extension string) bool { + switch strings.ToLower(extension) { + case ".azw3", ".azw", ".mobi": + return true + default: + return false + } +} + +func extractCandidate(candidate bookCandidate) (BookResult, error) { + sidecarPaths, err := findSidecarFiles(candidate.sidecarDir) + if err != nil { + return BookResult{}, err + } + merged := Sidecar{} + seenAnnotations := make(map[string]struct{}) + for _, path := range sidecarPaths { + data, err := os.ReadFile(path) + if err != nil { + return BookResult{}, fmt.Errorf("read %s: %w", filepath.Base(path), err) + } + decoded, err := DecodeSidecar(data) + if err != nil { + return BookResult{}, fmt.Errorf("decode %s: %w", filepath.Base(path), err) + } + if len(merged.PageMap.Positions) == 0 && len(decoded.PageMap.Positions) > 0 { + merged.PageMap = decoded.PageMap + } + for _, annotation := range decoded.Annotations { + key := fmt.Sprintf("%s\x00%d\x00%d\x00%d\x00%s", annotation.Type, + annotation.StartPosition, annotation.EndPosition, annotation.CreationTime.UnixMilli(), annotation.Note) + if _, duplicate := seenAnnotations[key]; duplicate { + continue + } + seenAnnotations[key] = struct{}{} + merged.Annotations = append(merged.Annotations, annotation) + } + } + + bookData, err := os.ReadFile(candidate.bookPath) + if err != nil { + return BookResult{}, fmt.Errorf("read book: %w", err) + } + title, assembled, err := AssembleBook(bookData) + if err != nil { + return BookResult{}, fmt.Errorf("assemble %s: %w", filepath.Base(candidate.bookPath), err) + } + if title == "" { + title = strings.TrimSuffix(filepath.Base(candidate.bookPath), filepath.Ext(candidate.bookPath)) + } + + sort.SliceStable(merged.Annotations, func(i, j int) bool { + if merged.Annotations[i].StartPosition != merged.Annotations[j].StartPosition { + return merged.Annotations[i].StartPosition < merged.Annotations[j].StartPosition + } + if !merged.Annotations[i].CreationTime.Equal(merged.Annotations[j].CreationTime) { + return merged.Annotations[i].CreationTime.Before(merged.Annotations[j].CreationTime) + } + return merged.Annotations[i].Type < merged.Annotations[j].Type + }) + result := BookResult{BookPath: candidate.bookPath, SidecarDir: candidate.sidecarDir, Title: title} + for _, annotation := range merged.Annotations { + text, exact := recoverText(assembled, annotation.StartPosition, annotation.EndPosition) + if text == "" { + continue + } + result.Highlights = append(result.Highlights, Highlight{ + Title: title, Text: text, ExactText: exact, Type: annotation.Type, + StartPosition: annotation.StartPosition, EndPosition: annotation.EndPosition, + CreatedAt: annotation.CreationTime.UTC(), Note: annotation.Note, + PageAt: pageAt(merged.PageMap, annotation.StartPosition), + }) + } + return result, nil +} + +func findSidecarFiles(directory string) ([]string, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, fmt.Errorf("read sidecar directory: %w", err) + } + var paths []string + hasKFX := false + for _, entry := range entries { + if entry.IsDir() { + continue + } + switch strings.ToLower(filepath.Ext(entry.Name())) { + case ".azw3r": + paths = append(paths, filepath.Join(directory, entry.Name())) + case ".yjr": + hasKFX = true + } + } + if len(paths) == 0 { + if hasKFX { + return nil, fmt.Errorf("sidecar contains only KFX .yjr data, which is unsupported") + } + return nil, fmt.Errorf("sidecar contains no .azw3r file") + } + sort.Strings(paths) + return paths, nil +} diff --git a/internal/sdr/krds.go b/internal/sdr/krds.go new file mode 100644 index 0000000..1c13d6c --- /dev/null +++ b/internal/sdr/krds.go @@ -0,0 +1,387 @@ +package sdr + +import ( + "encoding/binary" + "fmt" + "math" + "time" +) + +var krdsSignature = []byte{0, 0, 0, 0, 0, 0x1a, 0xb1, 0x26} + +const ( + krdsBoolean int8 = 0 + krdsInt int8 = 1 + krdsLong int8 = 2 + krdsUTF int8 = 3 + krdsDouble int8 = 4 + krdsShort int8 = 5 + krdsFloat int8 = 6 + krdsByte int8 = 7 + krdsChar int8 = 9 + krdsObjectBegin int8 = -2 + krdsObjectEnd int8 = -1 +) + +const maxKRDSValues = 1_000_000 + +type krdsDecoder struct { + data []byte + off int +} + +// DecodeSidecar decodes the annotation and page-map portions of a Kindle +// reader-data-store file. +func DecodeSidecar(data []byte) (Sidecar, error) { + d := &krdsDecoder{data: data} + if len(data) < len(krdsSignature) || string(data[:len(krdsSignature)]) != string(krdsSignature) { + return Sidecar{}, fmt.Errorf("invalid KRDS signature") + } + d.off = len(krdsSignature) + first, err := d.next(nil) + if err != nil { + return Sidecar{}, fmt.Errorf("decode KRDS version: %w", err) + } + if n, ok := asInt64(first); !ok || n != 1 { + return Sidecar{}, fmt.Errorf("unsupported KRDS version %v", first) + } + countValue, err := d.next(nil) + if err != nil { + return Sidecar{}, fmt.Errorf("decode KRDS object count: %w", err) + } + count, ok := asCount(countValue) + if !ok { + return Sidecar{}, fmt.Errorf("invalid KRDS object count %v", countValue) + } + + var result Sidecar + for i := 0; i < count; i++ { + value, err := d.next(nil) + if err != nil { + return Sidecar{}, fmt.Errorf("decode KRDS object %d: %w", i, err) + } + object, ok := value.(map[string]any) + if !ok { + continue + } + if cache, ok := object["annotation.cache.object"].(map[string]any); ok { + appendAnnotations(&result, cache) + } + if apnx, ok := object["apnx.key"].(map[string]any); ok { + result.PageMap = pageMapFromObject(apnx) + } + } + return result, nil +} + +func (d *krdsDecoder) next(forced *int8) (any, error) { + var datatype int8 + if forced != nil { + datatype = *forced + } else { + b, err := d.byte() + if err != nil { + return nil, err + } + datatype = int8(b) + } + + switch datatype { + case krdsBoolean: + b, err := d.byte() + if err != nil { + return nil, err + } + if b > 1 { + return nil, fmt.Errorf("invalid boolean %d at offset %d", b, d.off-1) + } + return b == 1, nil + case krdsInt: + b, err := d.take(4) + if err != nil { + return nil, err + } + return int64(int32(binary.BigEndian.Uint32(b))), nil + case krdsLong: + b, err := d.take(8) + if err != nil { + return nil, err + } + return int64(binary.BigEndian.Uint64(b)), nil + case krdsUTF: + emptyType := krdsBoolean + empty, err := d.next(&emptyType) + if err != nil { + return nil, err + } + if empty.(bool) { + return "", nil + } + b, err := d.take(2) + if err != nil { + return nil, err + } + n := int(binary.BigEndian.Uint16(b)) + text, err := d.take(n) + if err != nil { + return nil, err + } + return string(text), nil + case krdsDouble: + b, err := d.take(8) + if err != nil { + return nil, err + } + return math.Float64frombits(binary.BigEndian.Uint64(b)), nil + case krdsShort: + b, err := d.take(2) + if err != nil { + return nil, err + } + return int64(int16(binary.BigEndian.Uint16(b))), nil + case krdsFloat: + b, err := d.take(4) + if err != nil { + return nil, err + } + return float64(math.Float32frombits(binary.BigEndian.Uint32(b))), nil + case krdsByte: + b, err := d.byte() + return int64(int8(b)), err + case krdsChar: + b, err := d.byte() + return string([]byte{b}), err + case krdsObjectBegin: + nameType := krdsUTF + nameValue, err := d.next(&nameType) + if err != nil { + return nil, fmt.Errorf("decode object name: %w", err) + } + name := nameValue.(string) + values := make([]any, 0, 8) + for { + if d.off >= len(d.data) { + return nil, fmt.Errorf("unterminated object %q", name) + } + if int8(d.data[d.off]) == krdsObjectEnd { + d.off++ + break + } + if len(values) >= maxKRDSValues { + return nil, fmt.Errorf("object %q exceeds value limit", name) + } + value, err := d.next(nil) + if err != nil { + return nil, fmt.Errorf("decode object %q: %w", name, err) + } + values = append(values, value) + } + return map[string]any{name: decodeKRDSObject(name, values)}, nil + default: + return nil, fmt.Errorf("unknown KRDS datatype %d at offset %d", datatype, d.off-1) + } +} + +func (d *krdsDecoder) byte() (byte, error) { + b, err := d.take(1) + if err != nil { + return 0, err + } + return b[0], nil +} + +func (d *krdsDecoder) take(n int) ([]byte, error) { + if n < 0 || d.off > len(d.data)-n { + return nil, fmt.Errorf("truncated KRDS data at offset %d", d.off) + } + b := d.data[d.off : d.off+n] + d.off += n + return b, nil +} + +func decodeKRDSObject(name string, values []any) any { + pop := func() (any, bool) { + if len(values) == 0 { + return nil, false + } + v := values[0] + values = values[1:] + return v, true + } + + switch name { + case "saved.avl.interval.tree": + countValue, ok := pop() + count, valid := asCount(countValue) + if !ok || !valid || count > len(values) { + return values + } + return append([]any(nil), values[:count]...) + case "annotation.personal.bookmark", "annotation.personal.highlight", + "annotation.personal.note", "annotation.personal.clip_article", + "annotation.personal.handwritten_note", "annotation.personal.sticky_note", + "annotation.personal.underline": + if len(values) < 5 { + return values + } + obj := map[string]any{ + "startPosition": values[0], + "endPosition": values[1], + "creationTime": values[2], + "lastModificationTime": values[3], + "template": values[4], + } + if name == "annotation.personal.note" && len(values) > 5 { + obj["note"] = values[5] + } + return obj + case "annotation.cache.object": + return decodeAnnotationCache(values) + case "apnx.key": + return decodeAPNX(values) + default: + return values + } +} + +func decodeAnnotationCache(values []any) map[string]any { + result := make(map[string]any) + if len(values) == 0 { + return result + } + count, ok := asCount(values[0]) + if !ok { + return result + } + values = values[1:] + classes := map[int64]string{ + 0: "bookmark", 1: "highlight", 2: "note", 3: "clip_article", + 10: "handwritten_note", 11: "sticky_note", 13: "underline", + } + for i := 0; i < count && len(values) >= 2; i++ { + kindCode, ok := asInt64(values[0]) + tree, treeOK := objectValue(values[1], "saved.avl.interval.tree") + values = values[2:] + kind, known := classes[kindCode] + if !ok || !treeOK || !known { + continue + } + items, ok := tree.([]any) + if !ok { + continue + } + decoded := make([]map[string]any, 0, len(items)) + fullName := "annotation.personal." + kind + for _, item := range items { + if value, ok := objectValue(item, fullName); ok { + if obj, ok := value.(map[string]any); ok { + decoded = append(decoded, obj) + } + } + } + result[kind] = decoded + } + return result +} + +func decodeAPNX(values []any) map[string]any { + result := make(map[string]any) + if len(values) < 4 { + return result + } + result["asin"] = values[0] + result["cdeType"] = values[1] + result["sidecarAvailable"] = values[2] + count, ok := asCount(values[3]) + if !ok || len(values) < 4+count { + return result + } + positions := make([]int64, 0, count) + for _, value := range values[4 : 4+count] { + if position, ok := asPosition(value); ok { + positions = append(positions, position) + } + } + result["oPNToPosition"] = positions + return result +} + +func appendAnnotations(result *Sidecar, cache map[string]any) { + for _, kind := range []AnnotationType{AnnotationHighlight, AnnotationNote, AnnotationUnderline} { + items, ok := cache[string(kind)].([]map[string]any) + if !ok { + continue + } + for _, item := range items { + start, startOK := asPosition(item["startPosition"]) + end, endOK := asPosition(item["endPosition"]) + created, createdOK := asMilliseconds(item["creationTime"]) + if !startOK || !endOK || !createdOK { + continue + } + modified, _ := asMilliseconds(item["lastModificationTime"]) + note, _ := item["note"].(string) + result.Annotations = append(result.Annotations, Annotation{ + Type: kind, StartPosition: start, EndPosition: end, + CreationTime: created, ModificationTime: modified, Note: note, + }) + } + } +} + +func pageMapFromObject(obj map[string]any) PageMap { + positions, ok := obj["oPNToPosition"].([]int64) + if !ok { + return PageMap{} + } + return PageMap{Positions: append([]int64(nil), positions...)} +} + +func objectValue(value any, name string) (any, bool) { + obj, ok := value.(map[string]any) + if !ok { + return nil, false + } + v, ok := obj[name] + return v, ok +} + +func asCount(value any) (int, bool) { + n, ok := asInt64(value) + if !ok || n < 0 || n > maxKRDSValues { + return 0, false + } + return int(n), true +} + +func asInt64(value any) (int64, bool) { + switch v := value.(type) { + case int64: + return v, true + case int: + return int64(v), true + default: + return 0, false + } +} + +func asPosition(value any) (int64, bool) { + if n, ok := asInt64(value); ok { + return n, true + } + if text, ok := value.(string); ok { + var n int64 + if _, err := fmt.Sscan(text, &n); err == nil { + return n, true + } + } + return 0, false +} + +func asMilliseconds(value any) (time.Time, bool) { + n, ok := asInt64(value) + if !ok { + return time.Time{}, false + } + return time.UnixMilli(n).UTC(), true +} diff --git a/internal/sdr/sdr_test.go b/internal/sdr/sdr_test.go new file mode 100644 index 0000000..4b9ba4f --- /dev/null +++ b/internal/sdr/sdr_test.go @@ -0,0 +1,352 @@ +package sdr + +import ( + "bytes" + "encoding/binary" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestDecodeSidecarAndRecoverText(t *testing.T) { + created := time.Date(2026, 5, 7, 13, 59, 53, 233_000_000, time.UTC) + data := buildKRDSFixture(t, []fixtureAnnotation{ + {kind: 1, start: "3", end: "8", created: created}, + {kind: 2, start: "9", end: "14", created: created.Add(time.Minute), note: "remember this"}, + {kind: 13, start: "15", end: "20", created: created.Add(2 * time.Minute)}, + }, []int64{0, 9, 20}) + + sidecar, err := DecodeSidecar(data) + if err != nil { + t.Fatalf("DecodeSidecar() error = %v", err) + } + if len(sidecar.Annotations) != 3 { + t.Fatalf("got %d annotations, want 3", len(sidecar.Annotations)) + } + if sidecar.Annotations[1].Note != "remember this" { + t.Fatalf("note = %q", sidecar.Annotations[1].Note) + } + if got := pageAt(sidecar.PageMap, 12); got != "#1" { + t.Fatalf("pageAt() = %q, want #1", got) + } + if got := pageAt(PageMap{}, 12); got != "#12" { + t.Fatalf("pageAt() fallback = %q, want #12", got) + } + + markup := []byte(`

one & two three

`) + start := int64(bytes.Index(markup, []byte("one")) + 1) + end := int64(bytes.Index(markup, []byte("two")) + 2) + readable, exact := recoverText(markup, start, end) + if readable != "one & two" { + t.Fatalf("recoverText() readable = %q", readable) + } + if exact == readable || exact == "" { + t.Fatalf("recoverText() exact = %q, want an unsnapped non-empty slice", exact) + } +} + +func TestDecodeSidecarRejectsTruncatedData(t *testing.T) { + if _, err := DecodeSidecar(append([]byte(nil), krdsSignature...)); err == nil { + t.Fatal("DecodeSidecar() accepted truncated data") + } +} + +func TestPalmDOCDecompression(t *testing.T) { + // "hello hello": literal "hello", encoded space+letter, then a back-reference. + input := []byte{5, 'h', 'e', 'l', 'l', 'o', 0xe8, 0x80, 0x31} + got, err := decompressPalmDOC(input) + if err != nil { + t.Fatalf("decompressPalmDOC() error = %v", err) + } + if string(got) != "hello hello" { + t.Fatalf("decompressPalmDOC() = %q", got) + } +} + +func TestHUFFDecompression(t *testing.T) { + huff := make([]byte, 24+256*4+64*4) + copy(huff[:8], []byte("HUFF\x00\x00\x00\x18")) + binary.BigEndian.PutUint32(huff[8:12], 24) + binary.BigEndian.PutUint32(huff[12:16], uint32(24+256*4)) + for i := 0; i < 256; i++ { + binary.BigEndian.PutUint32(huff[24+i*4:28+i*4], 0x181) + } + cdic := make([]byte, 26) + copy(cdic[:8], []byte("CDIC\x00\x00\x00\x10")) + binary.BigEndian.PutUint32(cdic[8:12], 2) + binary.BigEndian.PutUint32(cdic[12:16], 1) + binary.BigEndian.PutUint16(cdic[16:18], 4) + binary.BigEndian.PutUint16(cdic[18:20], 7) + binary.BigEndian.PutUint16(cdic[20:22], 0x8001) + cdic[22] = 'A' + binary.BigEndian.PutUint16(cdic[23:25], 0x8001) + cdic[25] = 'B' + + decoder := &huffDecoder{} + if err := decoder.loadHUFF(huff); err != nil { + t.Fatalf("loadHUFF() error = %v", err) + } + if err := decoder.loadCDIC(cdic); err != nil { + t.Fatalf("loadCDIC() error = %v", err) + } + got, err := decoder.unpack([]byte{0xff}, 0) + if err != nil { + t.Fatalf("unpack() error = %v", err) + } + if string(got) != "AAAAAAAA" { + t.Fatalf("HUFF unpack = %q", got) + } +} + +func TestKF8IndexAssembly(t *testing.T) { + flow := []byte("

hello") + skeletonMain, skeletonExtra := buildIndexFixture(t, []indexTag{ + {tag: 1, valuesPerEntry: 1, mask: 1}, + {tag: 6, valuesPerEntry: 2, mask: 2}, + }, "s", 3, []uint64{1, 0, 7}) + fragmentMain, fragmentExtra := buildIndexFixture(t, []indexTag{ + {tag: 6, valuesPerEntry: 2, mask: 1}, + }, "3", 1, []uint64{7, 5}) + db := &palmDatabase{sections: [][]byte{flow, skeletonMain, skeletonExtra, fragmentMain, fragmentExtra}} + book := &mobiBook{db: db, skeleton: 1, fragment: 3, fdst: missingSection} + got, err := book.assembleKF8(flow) + if err != nil { + t.Fatalf("assembleKF8() error = %v", err) + } + if string(got) != "

hello

" { + t.Fatalf("assembleKF8() = %q", got) + } +} + +func TestAssembleBookRejectsDRMAndMobi7(t *testing.T) { + markup := []byte("

text

") + drm := buildKF8Fixture(t, "DRM", markup) + headerOffset := int(binary.BigEndian.Uint32(drm[78:82])) + binary.BigEndian.PutUint16(drm[headerOffset+12:headerOffset+14], 1) + if _, _, err := AssembleBook(drm); err == nil || !strings.Contains(err.Error(), "DRM") { + t.Fatalf("AssembleBook(DRM) error = %v", err) + } + + mobi7 := buildKF8Fixture(t, "Old", markup) + headerOffset = int(binary.BigEndian.Uint32(mobi7[78:82])) + binary.BigEndian.PutUint32(mobi7[headerOffset+36:headerOffset+40], 6) + if _, _, err := AssembleBook(mobi7); err == nil || !strings.Contains(err.Error(), "Mobi7") { + t.Fatalf("AssembleBook(Mobi7) error = %v", err) + } +} + +func TestExtractPathEndToEnd(t *testing.T) { + dir := t.TempDir() + bookPath := filepath.Join(dir, "Example.azw3") + sidecarDir := filepath.Join(dir, "Example.sdr") + if err := os.Mkdir(sidecarDir, 0o755); err != nil { + t.Fatal(err) + } + markup := []byte(`

alpha beta gamma

`) + if err := os.WriteFile(bookPath, buildKF8Fixture(t, "Example Title", markup), 0o600); err != nil { + t.Fatal(err) + } + start := int64(bytes.Index(markup, []byte("beta"))) + created := time.Date(2026, 7, 1, 8, 30, 0, 0, time.UTC) + sidecar := buildKRDSFixture(t, []fixtureAnnotation{{kind: 1, start: itoa64(start), end: itoa64(start + 4), created: created}}, []int64{0, start}) + if err := os.WriteFile(filepath.Join(sidecarDir, "annotations.azw3r"), sidecar, 0o600); err != nil { + t.Fatal(err) + } + + report, err := ExtractPath(dir) + if err != nil { + t.Fatalf("ExtractPath() error = %v; warnings = %v", err, report.Warnings) + } + if report.Decoded != 1 || len(report.Books) != 1 || len(report.Books[0].Highlights) != 1 { + t.Fatalf("unexpected report: %+v", report) + } + highlight := report.Books[0].Highlights[0] + if highlight.Title != "Example Title" || highlight.Text != "beta" || highlight.PageAt != "#1" { + t.Fatalf("unexpected highlight: %+v", highlight) + } + if !highlight.CreatedAt.Equal(created) { + t.Fatalf("createdAt = %s, want %s", highlight.CreatedAt, created) + } + + direct, err := ExtractPath(sidecarDir) + if err != nil || direct.Decoded != 1 { + t.Fatalf("direct ExtractPath() = %+v, %v", direct, err) + } +} + +type fixtureAnnotation struct { + kind int64 + start string + end string + created time.Time + note string +} + +func buildKRDSFixture(t *testing.T, annotations []fixtureAnnotation, pages []int64) []byte { + t.Helper() + var output bytes.Buffer + output.Write(krdsSignature) + writeKRDSInt(&output, 1) + writeKRDSInt(&output, 2) + + writeKRDSObjectStart(&output, "annotation.cache.object") + writeKRDSInt(&output, int64(len(annotations))) + for _, annotation := range annotations { + writeKRDSInt(&output, annotation.kind) + writeKRDSObjectStart(&output, "saved.avl.interval.tree") + writeKRDSInt(&output, 1) + name := map[int64]string{1: "highlight", 2: "note", 13: "underline"}[annotation.kind] + writeKRDSObjectStart(&output, "annotation.personal."+name) + writeKRDSString(&output, annotation.start) + writeKRDSString(&output, annotation.end) + writeKRDSLong(&output, annotation.created.UnixMilli()) + writeKRDSLong(&output, annotation.created.UnixMilli()) + writeKRDSString(&output, "template") + if annotation.kind == 2 { + writeKRDSString(&output, annotation.note) + } + output.WriteByte(0xff) + output.WriteByte(0xff) + } + output.WriteByte(0xff) + + writeKRDSObjectStart(&output, "apnx.key") + writeKRDSString(&output, "asin") + writeKRDSString(&output, "EBOK") + writeKRDSBool(&output, true) + writeKRDSInt(&output, int64(len(pages))) + for _, page := range pages { + writeKRDSLong(&output, page) + } + output.WriteByte(0xff) + return output.Bytes() +} + +func writeKRDSObjectStart(output *bytes.Buffer, name string) { + output.WriteByte(0xfe) + writeKRDSStringValue(output, name) +} + +func writeKRDSString(output *bytes.Buffer, value string) { + output.WriteByte(byte(krdsUTF)) + writeKRDSStringValue(output, value) +} + +func writeKRDSStringValue(output *bytes.Buffer, value string) { + output.WriteByte(0) + _ = binary.Write(output, binary.BigEndian, uint16(len(value))) + output.WriteString(value) +} + +func writeKRDSInt(output *bytes.Buffer, value int64) { + output.WriteByte(byte(krdsInt)) + _ = binary.Write(output, binary.BigEndian, int32(value)) +} + +func writeKRDSLong(output *bytes.Buffer, value int64) { + output.WriteByte(byte(krdsLong)) + _ = binary.Write(output, binary.BigEndian, value) +} + +func writeKRDSBool(output *bytes.Buffer, value bool) { + output.WriteByte(byte(krdsBoolean)) + if value { + output.WriteByte(1) + } else { + output.WriteByte(0) + } +} + +func buildKF8Fixture(t *testing.T, title string, markup []byte) []byte { + t.Helper() + header := make([]byte, 0x118+len(title)) + binary.BigEndian.PutUint16(header[0:2], mobiCompressionNone) + binary.BigEndian.PutUint16(header[8:10], 1) + copy(header[16:20], []byte("MOBI")) + binary.BigEndian.PutUint32(header[20:24], 0x108) + binary.BigEndian.PutUint32(header[28:32], 65001) + binary.BigEndian.PutUint32(header[36:40], 8) + binary.BigEndian.PutUint32(header[0x54:0x58], 0x118) + binary.BigEndian.PutUint32(header[0x58:0x5c], uint32(len(title))) + binary.BigEndian.PutUint32(header[0x70:0x74], missingSection) + binary.BigEndian.PutUint32(header[0xc0:0xc4], missingSection) + binary.BigEndian.PutUint32(header[0xc4:0xc8], 1) + binary.BigEndian.PutUint32(header[0xf8:0xfc], missingSection) + binary.BigEndian.PutUint32(header[0xfc:0x100], missingSection) + copy(header[0x118:], []byte(title)) + + sections := [][]byte{header, markup} + recordTableEnd := 78 + len(sections)*8 + total := recordTableEnd + for _, section := range sections { + total += len(section) + } + file := make([]byte, total) + copy(file[:32], []byte("Fixture")) + copy(file[60:68], []byte("BOOKMOBI")) + binary.BigEndian.PutUint16(file[76:78], uint16(len(sections))) + offset := recordTableEnd + for i, section := range sections { + binary.BigEndian.PutUint32(file[78+i*8:82+i*8], uint32(offset)) + copy(file[offset:], section) + offset += len(section) + } + return file +} + +func buildIndexFixture(t *testing.T, tags []indexTag, text string, control byte, values []uint64) ([]byte, []byte) { + t.Helper() + tagLength := 12 + len(tags)*4 + main := make([]byte, 56+tagLength) + copy(main[:4], []byte("INDX")) + binary.BigEndian.PutUint32(main[4:8], 56) + binary.BigEndian.PutUint32(main[24:28], 1) + copy(main[56:60], []byte("TAGX")) + binary.BigEndian.PutUint32(main[60:64], uint32(tagLength)) + binary.BigEndian.PutUint32(main[64:68], 1) + for i, tag := range tags { + position := 68 + i*4 + main[position] = tag.tag + main[position+1] = tag.valuesPerEntry + main[position+2] = tag.mask + main[position+3] = tag.endFlag + } + + entry := []byte{byte(len(text))} + entry = append(entry, text...) + entry = append(entry, control) + for _, value := range values { + if value >= 0x80 { + t.Fatalf("fixture value %d requires multi-byte VWI", value) + } + entry = append(entry, byte(value)|0x80) + } + const entryStart = 56 + idxt := entryStart + len(entry) + extra := make([]byte, idxt+6) + copy(extra[:4], []byte("INDX")) + binary.BigEndian.PutUint32(extra[4:8], 56) + binary.BigEndian.PutUint32(extra[20:24], uint32(idxt)) + binary.BigEndian.PutUint32(extra[24:28], 1) + copy(extra[entryStart:], entry) + copy(extra[idxt:idxt+4], []byte("IDXT")) + binary.BigEndian.PutUint16(extra[idxt+4:idxt+6], entryStart) + return main, extra +} + +func TestDiscoverSkipsKFXWithWarning(t *testing.T) { + dir := t.TempDir() + sidecar := filepath.Join(dir, "KFX.sdr") + if err := os.Mkdir(sidecar, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sidecar, "data.yjr"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + report, err := ExtractPath(dir) + if err == nil || !strings.Contains(err.Error(), "no supported") || len(report.Warnings) == 0 { + t.Fatalf("ExtractPath() report=%+v err=%v", report, err) + } +} diff --git a/internal/sdr/text.go b/internal/sdr/text.go new file mode 100644 index 0000000..7bb1c87 --- /dev/null +++ b/internal/sdr/text.go @@ -0,0 +1,117 @@ +package sdr + +import ( + "html" + "regexp" + "sort" + "strings" +) + +const maxWordSnap = 40 + +var ( + markupTagPattern = regexp.MustCompile(`<[^>]*>`) + spacePattern = regexp.MustCompile(`\s+`) +) + +func recoverText(assembled []byte, start, end int64) (string, string) { + if start > end { + start, end = end, start + } + start = clampPosition(start, len(assembled)) + end = clampPosition(end, len(assembled)) + exact := cleanMarkup(assembled[start:end]) + left, right := snapRange(assembled, int(start), int(end)) + return cleanMarkup(assembled[left:right]), exact +} + +func clampPosition(position int64, length int) int64 { + if position < 0 { + return 0 + } + if position > int64(length) { + return int64(length) + } + return position +} + +func snapRange(data []byte, start, end int) (int, int) { + left := start + steps := 0 + for left > 0 && !asciiWhitespace(data[left-1]) && steps < maxWordSnap { + left-- + steps++ + } + if steps >= maxWordSnap { + left = start + } + right := end + steps = 0 + for right < len(data) && !asciiWhitespace(data[right]) && steps < maxWordSnap { + right++ + steps++ + } + if steps >= maxWordSnap { + right = end + } + return left, right +} + +func asciiWhitespace(b byte) bool { + switch b { + case ' ', '\t', '\n', '\r', '\f', '\v': + return true + default: + return false + } +} + +func cleanMarkup(data []byte) string { + text := string(data) + text = markupTagPattern.ReplaceAllString(text, " ") + if close := strings.IndexByte(text, '>'); close >= 0 { + open := strings.IndexByte(text, '<') + if open < 0 || close < open { + text = text[close+1:] + } + } + if open := strings.LastIndexByte(text, '<'); open >= 0 && !strings.Contains(text[open:], ">") { + text = text[:open] + } + text = html.UnescapeString(text) + return strings.TrimSpace(spacePattern.ReplaceAllString(text, " ")) +} + +func pageAt(pageMap PageMap, position int64) string { + if len(pageMap.Positions) > 0 { + page := sort.Search(len(pageMap.Positions), func(i int) bool { + return pageMap.Positions[i] > position + }) - 1 + if page >= 0 { + return "#" + itoa64(int64(page)) + } + } + return "#" + itoa64(position) +} + +func itoa64(value int64) string { + if value == 0 { + return "0" + } + negative := value < 0 + if negative { + value = -value + } + var buf [20]byte + i := len(buf) + for value > 0 { + i-- + buf[i] = byte('0' + value%10) + value /= 10 + } + if negative { + i-- + buf[i] = '-' + } + return string(buf[i:]) +} diff --git a/internal/sdr/types.go b/internal/sdr/types.go new file mode 100644 index 0000000..bd053b4 --- /dev/null +++ b/internal/sdr/types.go @@ -0,0 +1,61 @@ +package sdr + +import "time" + +// AnnotationType identifies a Kindle text annotation. +type AnnotationType string + +const ( + AnnotationHighlight AnnotationType = "highlight" + AnnotationNote AnnotationType = "note" + AnnotationUnderline AnnotationType = "underline" +) + +// Annotation is the subset of a KRDS annotation used by the extractor. +type Annotation struct { + Type AnnotationType + StartPosition int64 + EndPosition int64 + CreationTime time.Time + ModificationTime time.Time + Note string +} + +// PageMap maps assembled-text positions to printed pages. +type PageMap struct { + Positions []int64 +} + +// Sidecar is the annotation data decoded from one or more .azw3r files. +type Sidecar struct { + Annotations []Annotation + PageMap PageMap +} + +// Highlight is a recovered text annotation and its display metadata. +type Highlight struct { + Title string + Text string + ExactText string + Type AnnotationType + StartPosition int64 + EndPosition int64 + CreatedAt time.Time + Note string + PageAt string +} + +// BookResult is the extraction result for one Kindle book. +type BookResult struct { + BookPath string + SidecarDir string + Title string + Highlights []Highlight +} + +// Report summarizes a path scan. +type Report struct { + Books []BookResult + Warnings []string + Decoded int +} From 49240b3b7706b9bc95cef37c77c2107e9df53be0 Mon Sep 17 00:00:00 2001 From: AnnatarHe Date: Sun, 2 Aug 2026 10:00:55 +0800 Subject: [PATCH 2/3] feat(cli): support KFX SDR highlights --- .gitignore | 5 +- README.md | 25 +- go.mod | 1 + go.sum | 18 ++ internal/commands/sdr.go | 9 +- internal/sdr/extract.go | 126 +++++++-- internal/sdr/kfx.go | 568 +++++++++++++++++++++++++++++++++++++++ internal/sdr/krds.go | 8 +- internal/sdr/sdr_test.go | 196 +++++++++++++- internal/sdr/types.go | 2 +- 10 files changed, 917 insertions(+), 41 deletions(-) create mode 100644 internal/sdr/kfx.go diff --git a/.gitignore b/.gitignore index ac894d8..f43c8a8 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,7 @@ dist/ #Added by cargo /target -.DS_Store \ No newline at end of file +.DS_Store + +# Local Kindle books may contain copyrighted text and signed delivery metadata. +/fixtures/moto-jurnory/ diff --git a/README.md b/README.md index cd41cfc..f8cf947 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ ck-cli sdr --path "/path/to/Book.sdr" --json ### Kindle `.sdr` Highlights Recent Kindle sidecars store annotations as positions rather than embedding the -highlighted words. The `sdr` command pairs each `.azw3r` sidecar with its sibling -AZW3/KF8 book and reconstructs the selected text locally: +highlighted words. The `sdr` command pairs `.azw3r` and `.yjr` sidecars with +their sibling AZW3/KF8 or KFX books and reconstructs the selected text locally: ```bash # Recursively scan a Kindle root or documents directory @@ -56,18 +56,20 @@ ck-cli sdr --path "/Volumes/Kindle/documents" # Process a single sidecar directory or book ck-cli sdr --path "/path/to/Book.sdr" ck-cli sdr --path "/path/to/Book.azw3" --json +ck-cli sdr --path "/path/to/Book.kfx" --json ``` `--path` accepts a Kindle root/documents tree, one `.sdr` directory, or one -`.azw3`, `.azw`, or KF8-containing `.mobi` file. The default output is readable -text grouped by book. `--json` emits the same `title`, `content`, `pageAt`, and -`createdAt` schema as `parse`; printed APNX pages are preferred, with the raw -annotation position used as a fallback. +`.azw3`, `.azw`, KF8-containing `.mobi`, or `.kfx` file. The default output is +readable text grouped by book. `--json` emits the same `title`, `content`, +`pageAt`, and `createdAt` schema as `parse`; printed APNX or KFX navigation pages +are preferred, with the raw annotation position used as a fallback. The implementation is read-only, offline, and written natively in Go—Python and -KindleUnpack are not runtime dependencies. It supports unencrypted AZW3/KF8 books -with `.azw3r` sidecars. DRM-protected books, Mobi7, and KFX/`.yjr` are skipped as -unsupported. +KindleUnpack are not runtime dependencies. It supports unencrypted AZW3/KF8 +books with `.azw3r` sidecars and unencrypted KFX books with `.yjr` sidecars. +DRM-protected books and Mobi7 remain unsupported; `.yjf` reading statistics are +ignored because they do not contain highlight selections. ### Web Sync @@ -101,7 +103,7 @@ See [Makefile](./Makefile) for all commands. - Flexible I/O (files, stdin/stdout, web sync) - High-performance processing of large files - Direct ClippingKK web service integration -- Native Kindle `.sdr`/`.azw3r` highlight extraction +- Native Kindle `.sdr` highlight extraction for `.azw3r` and `.yjr` - Cross-platform (macOS, Linux, Windows) ## Contributing @@ -117,3 +119,6 @@ The `.sdr` implementation was informed by the published format research in and the container behavior documented by [KindleUnpack](https://github.com/kevinhendricks/KindleUnpack). No code from either GPLv3 project is bundled or required. + +Binary Amazon Ion values in KFX containers are decoded with the Apache-2.0 +licensed [Amazon Ion Go](https://github.com/amazon-ion/ion-go) library. diff --git a/go.mod b/go.mod index aa4778c..ff18ed3 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/clippingkk/cli go 1.24 require ( + github.com/amazon-ion/ion-go v1.5.0 github.com/pelletier/go-toml/v2 v2.2.4 github.com/urfave/cli/v2 v2.27.7 ) diff --git a/go.sum b/go.sum index 545c9f7..bde8e81 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,28 @@ +github.com/amazon-ion/ion-go v1.5.0 h1:fxsAyFda8N9HsM2xYbQSxJ3Qi/oLn0xzLoiXWG3bseg= +github.com/amazon-ion/ion-go v1.5.0/go.mod h1:3ZEje8i20TiIPVZlN+KE3B2ppZ1B8d9F/KaT7Dtec+k= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/commands/sdr.go b/internal/commands/sdr.go index 5ae1b35..67574eb 100644 --- a/internal/commands/sdr.go +++ b/internal/commands/sdr.go @@ -17,18 +17,19 @@ var SDRCommand = &cli.Command{ Name: "sdr", Usage: "Extract highlighted text from Kindle .sdr sidecars", Description: `Read Kindle .sdr sidecars and recover highlighted text from their -unencrypted AZW3/KF8 books. The path may be a mounted Kindle or documents tree, -a single .sdr directory, or a single AZW3/KF8 book. +unencrypted AZW3/KF8 or KFX books. The path may be a mounted Kindle or documents +tree, a single .sdr directory, or a supported book. Examples: ck-cli sdr --path /Volumes/Kindle/documents ck-cli sdr --path "Book.sdr" --json - ck-cli sdr --path "Book.azw3" --json`, + ck-cli sdr --path "Book.azw3" --json + ck-cli sdr --path "Book.kfx" --json`, Flags: []cli.Flag{ &cli.StringFlag{ Name: "path", Aliases: []string{"p"}, - Usage: "Kindle documents tree, .sdr directory, or AZW3/KF8 book", + Usage: "Kindle documents tree, .sdr directory, or AZW3/KF8/KFX book", Required: true, }, &cli.BoolFlag{ diff --git a/internal/sdr/extract.go b/internal/sdr/extract.go index 9b69d05..474f776 100644 --- a/internal/sdr/extract.go +++ b/internal/sdr/extract.go @@ -44,9 +44,9 @@ func ExtractPath(path string) (Report, error) { } if report.Decoded == 0 { if len(report.Warnings) > 0 { - return report, fmt.Errorf("no supported AZW3/KF8 sidecar pairs could be decoded") + return report, fmt.Errorf("no supported Kindle sidecar pairs could be decoded") } - return report, fmt.Errorf("no supported AZW3/KF8 sidecar pairs found under %s", path) + return report, fmt.Errorf("no supported Kindle sidecar pairs found under %s", path) } return report, nil } @@ -108,7 +108,8 @@ func findSiblingBook(sidecarDir string) (string, error) { return "", fmt.Errorf("read book directory: %w", err) } baseName := filepath.Base(base) - priority := map[string]int{".azw3": 0, ".azw": 1, ".mobi": 2} + hasAZW3R, hasYJR := sidecarFormats(sidecarDir) + priority := map[string]int{".azw3": 0, ".azw": 1, ".mobi": 2, ".kfx": 3} type match struct { path string priority int @@ -123,10 +124,16 @@ func findSiblingBook(sidecarDir string) (string, error) { if !ok || !strings.EqualFold(strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())), baseName) { continue } + if ext == ".kfx" && !hasYJR { + continue + } + if ext != ".kfx" && !hasAZW3R { + continue + } matches = append(matches, match{path: filepath.Join(parent, entry.Name()), priority: order}) } if len(matches) == 0 { - return "", fmt.Errorf("no sibling AZW3/KF8 book found (KFX and DRM are unsupported)") + return "", fmt.Errorf("no compatible sibling AZW3/KF8 or KFX book found") } sort.Slice(matches, func(i, j int) bool { if matches[i].priority != matches[j].priority { @@ -137,9 +144,28 @@ func findSiblingBook(sidecarDir string) (string, error) { return matches[0].path, nil } +func sidecarFormats(directory string) (hasAZW3R, hasYJR bool) { + entries, err := os.ReadDir(directory) + if err != nil { + return false, false + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + switch strings.ToLower(filepath.Ext(entry.Name())) { + case ".azw3r": + hasAZW3R = true + case ".yjr": + hasYJR = true + } + } + return hasAZW3R, hasYJR +} + func supportedBookExtension(extension string) bool { switch strings.ToLower(extension) { - case ".azw3", ".azw", ".mobi": + case ".azw3", ".azw", ".mobi", ".kfx": return true default: return false @@ -147,7 +173,8 @@ func supportedBookExtension(extension string) bool { } func extractCandidate(candidate bookCandidate) (BookResult, error) { - sidecarPaths, err := findSidecarFiles(candidate.sidecarDir) + bookExtension := strings.ToLower(filepath.Ext(candidate.bookPath)) + sidecarPaths, err := findSidecarFiles(candidate.sidecarDir, bookExtension) if err != nil { return BookResult{}, err } @@ -180,9 +207,20 @@ func extractCandidate(candidate bookCandidate) (BookResult, error) { if err != nil { return BookResult{}, fmt.Errorf("read book: %w", err) } - title, assembled, err := AssembleBook(bookData) - if err != nil { - return BookResult{}, fmt.Errorf("assemble %s: %w", filepath.Base(candidate.bookPath), err) + var title string + var assembled []byte + var kfx kfxBook + if bookExtension == ".kfx" { + kfx, err = assembleKFX(bookData) + if err != nil { + return BookResult{}, fmt.Errorf("assemble %s: %w", filepath.Base(candidate.bookPath), err) + } + title = kfx.title + } else { + title, assembled, err = AssembleBook(bookData) + if err != nil { + return BookResult{}, fmt.Errorf("assemble %s: %w", filepath.Base(candidate.bookPath), err) + } } if title == "" { title = strings.TrimSuffix(filepath.Base(candidate.bookPath), filepath.Ext(candidate.bookPath)) @@ -198,8 +236,24 @@ func extractCandidate(candidate bookCandidate) (BookResult, error) { return merged.Annotations[i].Type < merged.Annotations[j].Type }) result := BookResult{BookPath: candidate.bookPath, SidecarDir: candidate.sidecarDir, Title: title} - for _, annotation := range merged.Annotations { - text, exact := recoverText(assembled, annotation.StartPosition, annotation.EndPosition) + annotations := merged.Annotations + if bookExtension == ".kfx" { + annotations = associateNotes(annotations) + } + for _, annotation := range annotations { + var text, exact string + var page string + if bookExtension == ".kfx" { + var found bool + text, found = kfx.textAt(annotation.StartPosition, annotation.EndPosition) + if found { + exact = text + } + page = kfx.pageAt(annotation.StartPosition) + } else { + text, exact = recoverText(assembled, annotation.StartPosition, annotation.EndPosition) + page = pageAt(merged.PageMap, annotation.StartPosition) + } if text == "" { continue } @@ -207,35 +261,63 @@ func extractCandidate(candidate bookCandidate) (BookResult, error) { Title: title, Text: text, ExactText: exact, Type: annotation.Type, StartPosition: annotation.StartPosition, EndPosition: annotation.EndPosition, CreatedAt: annotation.CreationTime.UTC(), Note: annotation.Note, - PageAt: pageAt(merged.PageMap, annotation.StartPosition), + PageAt: page, }) } return result, nil } -func findSidecarFiles(directory string) ([]string, error) { +func associateNotes(annotations []Annotation) []Annotation { + notesByStart := make(map[int64][]int) + for index, annotation := range annotations { + if annotation.Type == AnnotationNote && annotation.Note != "" { + notesByStart[annotation.StartPosition] = append(notesByStart[annotation.StartPosition], index) + } + } + consumed := make(map[int]bool) + result := make([]Annotation, 0, len(annotations)) + for _, annotation := range annotations { + if annotation.Type == AnnotationHighlight || annotation.Type == AnnotationUnderline { + for _, noteIndex := range notesByStart[annotation.EndPosition] { + if annotation.Note == "" { + annotation.Note = annotations[noteIndex].Note + } else { + annotation.Note += "\n" + annotations[noteIndex].Note + } + consumed[noteIndex] = true + } + } + result = append(result, annotation) + } + filtered := result[:0] + for index, annotation := range result { + if !consumed[index] { + filtered = append(filtered, annotation) + } + } + return filtered +} + +func findSidecarFiles(directory, bookExtension string) ([]string, error) { entries, err := os.ReadDir(directory) if err != nil { return nil, fmt.Errorf("read sidecar directory: %w", err) } var paths []string - hasKFX := false + wanted := ".azw3r" + if bookExtension == ".kfx" { + wanted = ".yjr" + } for _, entry := range entries { if entry.IsDir() { continue } - switch strings.ToLower(filepath.Ext(entry.Name())) { - case ".azw3r": + if strings.ToLower(filepath.Ext(entry.Name())) == wanted { paths = append(paths, filepath.Join(directory, entry.Name())) - case ".yjr": - hasKFX = true } } if len(paths) == 0 { - if hasKFX { - return nil, fmt.Errorf("sidecar contains only KFX .yjr data, which is unsupported") - } - return nil, fmt.Errorf("sidecar contains no .azw3r file") + return nil, fmt.Errorf("sidecar contains no %s file", wanted) } sort.Strings(paths) return paths, nil diff --git a/internal/sdr/kfx.go b/internal/sdr/kfx.go new file mode 100644 index 0000000..b9798e0 --- /dev/null +++ b/internal/sdr/kfx.go @@ -0,0 +1,568 @@ +package sdr + +import ( + "bytes" + "encoding/binary" + "fmt" + "sort" + "strings" + "unicode/utf8" + + "github.com/amazon-ion/ion-go/ion" +) + +const ( + kfxContainerHeaderSize = 18 + kfxEntityHeaderSize = 10 + maxKFXContainerSize = 64 << 20 + maxKFXEntities = 100_000 + maxYJSymbolID = 2048 +) + +type kfxEntity struct { + id string + typeName string + value any +} + +type kfxSection struct { + position int64 + text string +} + +type kfxPage struct { + position int64 + label string +} + +type kfxPositionChunk struct { + position int64 + eid int64 + offset int64 + length int64 +} + +type kfxBook struct { + title string + sections []kfxSection + pages []kfxPage +} + +// assembleKFX decodes the text and navigation fragments needed to map Kindle +// annotation positions. It deliberately does not attempt to convert the book +// to another publication format. +func assembleKFX(data []byte) (kfxBook, error) { + entities, err := decodeKFXContainer(data) + if err != nil { + return kfxBook{}, err + } + book := kfxBook{} + textPools := make(map[string][]string) + sectionStarts := make(map[string]int64) + var positionEntities []kfxEntity + var contentEntities []kfxEntity + var navigationEntities []kfxEntity + for _, entity := range entities { + switch entity.typeName { + case "$145": + if pool := kfxStringList(kfxMap(entity.value)["$146"]); len(pool) > 0 { + textPools[entity.id] = pool + } + case "$259", "$260": + contentEntities = append(contentEntities, entity) + case "$265": + collectKFXSectionStarts(entity.value, sectionStarts) + case "$609": + positionEntities = append(positionEntities, entity) + case "$258", "$490", "$538": + if book.title == "" { + book.title = findKFXTitle(entity.value) + } + case "$391": + navigationEntities = append(navigationEntities, entity) + } + } + chunksByEID := make(map[int64][]kfxPositionChunk) + for _, entity := range positionEntities { + sectionName := kfxString(kfxMap(entity.value)["$174"]) + start, ok := sectionStarts[sectionName] + if !ok { + continue + } + chunks, err := decodeKFXPositionMap(kfxMap(entity.value)["$181"], start) + if err != nil { + return kfxBook{}, fmt.Errorf("decode KFX position map %s: %w", sectionName, err) + } + for _, chunk := range chunks { + chunksByEID[chunk.eid] = append(chunksByEID[chunk.eid], chunk) + } + } + for eid := range chunksByEID { + sort.Slice(chunksByEID[eid], func(i, j int) bool { + return chunksByEID[eid][i].offset < chunksByEID[eid][j].offset + }) + } + textOffsets := make(map[int64]int64) + for _, entity := range contentEntities { + collectKFXContent(entity.value, textPools, chunksByEID, textOffsets, &book.sections) + } + for _, entity := range navigationEntities { + collectKFXPages(entity.value, chunksByEID, &book.pages) + } + if len(book.sections) == 0 { + return kfxBook{}, fmt.Errorf("KFX container contains no readable text fragments") + } + sort.Slice(book.sections, func(i, j int) bool { return book.sections[i].position < book.sections[j].position }) + sort.Slice(book.pages, func(i, j int) bool { return book.pages[i].position < book.pages[j].position }) + return book, nil +} + +func decodeKFXContainer(data []byte) ([]kfxEntity, error) { + if len(data) < kfxContainerHeaderSize || string(data[:4]) != "CONT" { + return nil, fmt.Errorf("invalid KFX CONT signature") + } + if len(data) > maxKFXContainerSize { + return nil, fmt.Errorf("KFX container exceeds %d byte limit", maxKFXContainerSize) + } + version := binary.LittleEndian.Uint16(data[4:6]) + if version != 1 && version != 2 { + return nil, fmt.Errorf("unsupported KFX container version %d", version) + } + headerLength := uint64(binary.LittleEndian.Uint32(data[6:10])) + infoOffset := uint64(binary.LittleEndian.Uint32(data[10:14])) + infoLength := uint64(binary.LittleEndian.Uint32(data[14:18])) + if headerLength < kfxContainerHeaderSize || headerLength > uint64(len(data)) { + return nil, fmt.Errorf("invalid KFX header length %d", headerLength) + } + infoData, err := boundedKFXSlice(data, infoOffset, infoLength, "container info") + if err != nil { + return nil, err + } + infoValue, err := decodeKFXIon(infoData, nil) + if err != nil { + return nil, fmt.Errorf("decode KFX container info: %w", err) + } + info, ok := infoValue.(map[string]any) + if !ok { + return nil, fmt.Errorf("KFX container info is not a struct") + } + if compression, _ := kfxInt(info["$410"]); compression != 0 { + return nil, fmt.Errorf("unsupported KFX container compression %d", compression) + } + if drm, _ := kfxInt(info["$411"]); drm != 0 { + return nil, fmt.Errorf("DRM-protected KFX containers are unsupported") + } + indexOffset, indexOK := kfxUint(info["$413"]) + indexLength, lengthOK := kfxUint(info["$414"]) + if !indexOK || !lengthOK || indexLength%24 != 0 { + return nil, fmt.Errorf("invalid KFX entity index") + } + if indexLength/24 > maxKFXEntities { + return nil, fmt.Errorf("KFX entity index exceeds entry limit") + } + indexData, err := boundedKFXSlice(data, indexOffset, indexLength, "entity index") + if err != nil { + return nil, err + } + + var documentSymbols []byte + if symbolLength, ok := kfxUint(info["$416"]); ok && symbolLength > 0 { + symbolOffset, offsetOK := kfxUint(info["$415"]) + if !offsetOK { + return nil, fmt.Errorf("KFX document symbol table has no offset") + } + documentSymbols, err = boundedKFXSlice(data, symbolOffset, symbolLength, "document symbol table") + if err != nil { + return nil, err + } + } + symbols, err := kfxSymbolTable(documentSymbols) + if err != nil { + return nil, fmt.Errorf("decode KFX document symbols: %w", err) + } + + result := make([]kfxEntity, 0, indexLength/24) + for offset := 0; offset < len(indexData); offset += 24 { + idSID := uint64(binary.LittleEndian.Uint32(indexData[offset : offset+4])) + typeSID := uint64(binary.LittleEndian.Uint32(indexData[offset+4 : offset+8])) + entityOffset := binary.LittleEndian.Uint64(indexData[offset+8 : offset+16]) + entityLength := binary.LittleEndian.Uint64(indexData[offset+16 : offset+24]) + serialized, err := boundedKFXSlice(data, headerLength+entityOffset, entityLength, "entity") + if err != nil { + return nil, fmt.Errorf("KFX entity %d: %w", offset/24, err) + } + if len(serialized) < kfxEntityHeaderSize || string(serialized[:4]) != "ENTY" { + return nil, fmt.Errorf("KFX entity %d has invalid ENTY signature", offset/24) + } + if entityVersion := binary.LittleEndian.Uint16(serialized[4:6]); entityVersion != 1 { + return nil, fmt.Errorf("KFX entity %d has unsupported version %d", offset/24, entityVersion) + } + entityHeaderLength := uint64(binary.LittleEndian.Uint32(serialized[6:10])) + if entityHeaderLength < kfxEntityHeaderSize || entityHeaderLength > uint64(len(serialized)) { + return nil, fmt.Errorf("KFX entity %d has invalid header length", offset/24) + } + entityInfoData := serialized[kfxEntityHeaderSize:entityHeaderLength] + if len(entityInfoData) > 0 { + entityInfoValue, err := decodeKFXIon(entityInfoData, documentSymbols) + if err != nil { + return nil, fmt.Errorf("decode KFX entity %d info: %w", offset/24, err) + } + if entityInfo, ok := entityInfoValue.(map[string]any); ok { + if compression, _ := kfxInt(entityInfo["$410"]); compression != 0 { + return nil, fmt.Errorf("KFX entity %d uses unsupported compression %d", offset/24, compression) + } + if drm, _ := kfxInt(entityInfo["$411"]); drm != 0 { + return nil, fmt.Errorf("KFX entity %d is DRM-protected", offset/24) + } + } + } + typeName := kfxSymbolByID(symbols, typeSID) + var value any + if typeName == "$417" || typeName == "$418" { + value = append([]byte(nil), serialized[entityHeaderLength:]...) + } else { + value, err = decodeKFXIon(serialized[entityHeaderLength:], documentSymbols) + if err != nil { + return nil, fmt.Errorf("decode KFX entity %d payload: %w", offset/24, err) + } + } + result = append(result, kfxEntity{ + id: kfxSymbolByID(symbols, idSID), + typeName: typeName, + value: value, + }) + } + return result, nil +} + +func boundedKFXSlice(data []byte, offset, length uint64, label string) ([]byte, error) { + if offset > uint64(len(data)) || length > uint64(len(data))-offset { + return nil, fmt.Errorf("%s range is outside the KFX container", label) + } + return data[offset : offset+length], nil +} + +func yjSharedSymbols() ion.SharedSymbolTable { + symbols := make([]string, maxYJSymbolID-9) + for sid := 10; sid <= maxYJSymbolID; sid++ { + symbols[sid-10] = fmt.Sprintf("$%d", sid) + } + return ion.NewSharedSymbolTable("YJ_symbols", 10, symbols) +} + +func kfxIonPrelude() ([]byte, error) { + var output bytes.Buffer + w := ion.NewBinaryWriter(&output, yjSharedSymbols()) + if err := w.WriteInt(0); err != nil { + return nil, err + } + if err := w.Finish(); err != nil { + return nil, err + } + encoded := output.Bytes() + if len(encoded) == 0 || encoded[len(encoded)-1] != 0x20 { + return nil, fmt.Errorf("create KFX Ion symbol prelude") + } + return append([]byte(nil), encoded[:len(encoded)-1]...), nil +} + +func decodeKFXIon(data, documentSymbols []byte) (any, error) { + var stream []byte + if len(documentSymbols) > 0 { + stream = append(stream, 0xe0, 0x01, 0x00, 0xea) + stream = append(stream, stripIonVersion(documentSymbols)...) + } else { + prelude, err := kfxIonPrelude() + if err != nil { + return nil, err + } + stream = append(stream, prelude...) + } + stream = append(stream, stripIonVersion(data)...) + reader := ion.NewReaderCat(bytes.NewReader(stream), ion.NewCatalog(yjSharedSymbols())) + return ion.NewDecoder(reader).Decode() +} + +func stripIonVersion(data []byte) []byte { + if len(data) >= 4 && bytes.Equal(data[:4], []byte{0xe0, 0x01, 0x00, 0xea}) { + return data[4:] + } + return data +} + +func kfxSymbolTable(documentSymbols []byte) (ion.SymbolTable, error) { + var stream []byte + if len(documentSymbols) > 0 { + stream = append(stream, 0xe0, 0x01, 0x00, 0xea) + stream = append(stream, stripIonVersion(documentSymbols)...) + } else { + prelude, err := kfxIonPrelude() + if err != nil { + return nil, err + } + stream = append(stream, prelude...) + } + stream = append(stream, 0x20) + reader := ion.NewReaderCat(bytes.NewReader(stream), ion.NewCatalog(yjSharedSymbols())) + if !reader.Next() { + if reader.Err() != nil { + return nil, reader.Err() + } + return nil, fmt.Errorf("decode KFX symbol table") + } + return reader.SymbolTable(), nil +} + +func kfxSymbolByID(table ion.SymbolTable, sid uint64) string { + if table != nil { + if symbol, ok := table.FindByID(sid); ok { + return symbol + } + } + return fmt.Sprintf("$%d", sid) +} + +func kfxUint(value any) (uint64, bool) { + n, ok := kfxInt(value) + if !ok || n < 0 { + return 0, false + } + return uint64(n), true +} + +func kfxInt(value any) (int64, bool) { + switch v := value.(type) { + case int: + return int64(v), true + case int64: + return v, true + case uint64: + if v <= uint64(^uint64(0)>>1) { + return int64(v), true + } + } + return 0, false +} + +func kfxString(value any) string { + switch v := value.(type) { + case string: + return v + case *string: + if v != nil { + return *v + } + case *ion.SymbolToken: + if v != nil && v.Text != nil { + return *v.Text + } + } + return "" +} + +func kfxMap(value any) map[string]any { + result, _ := value.(map[string]any) + return result +} + +func kfxList(value any) []any { + result, _ := value.([]any) + return result +} + +func kfxStringList(value any) []string { + values := kfxList(value) + result := make([]string, 0, len(values)) + for _, value := range values { + if text := kfxString(value); text != "" { + result = append(result, text) + } else { + result = append(result, "") + } + } + return result +} + +func findKFXTitle(value any) string { + var title string + walkKFX(value, func(field string, child any) { + if title != "" { + return + } + if field == "$153" || strings.EqualFold(field, "title") { + title = kfxString(child) + return + } + if object := kfxMap(child); kfxString(object["$492"]) == "title" { + title = kfxString(object["$307"]) + } + }) + return title +} + +func collectKFXSectionStarts(value any, starts map[string]int64) { + for _, sectionValue := range kfxList(kfxMap(value)["$181"]) { + section := kfxMap(sectionValue) + name := kfxString(section["$174"]) + start, ok := kfxInt(section["$184"]) + if name != "" && ok && start >= 0 { + starts[name] = start + } + } +} + +func decodeKFXPositionMap(value any, sectionStart int64) ([]kfxPositionChunk, error) { + entries := kfxList(value) + var result []kfxPositionChunk + var position, eid, eidOffset int64 + for index, entry := range entries { + var nextPosition, nextEID, nextOffset int64 + if values := kfxList(entry); len(values) >= 2 && len(values) <= 3 { + positionDelta, positionOK := kfxInt(values[0]) + eidValue, eidOK := kfxInt(values[1]) + if !positionOK || !eidOK || positionDelta < 0 || eidValue < 0 { + return nil, fmt.Errorf("invalid list entry %d", index) + } + nextPosition = position + positionDelta + nextEID = eidValue + if len(values) == 3 { + var ok bool + nextOffset, ok = kfxInt(values[2]) + if !ok || nextOffset < 0 { + return nil, fmt.Errorf("invalid offset in entry %d", index) + } + } + } else if delta, ok := kfxInt(entry); ok && delta >= 0 { + nextPosition = position + delta + nextEID = eid + 1 + } else { + return nil, fmt.Errorf("invalid entry %d", index) + } + if index > 0 { + length := nextPosition - position + if length < 0 { + return nil, fmt.Errorf("position moved backwards at entry %d", index) + } + result = append(result, kfxPositionChunk{ + position: sectionStart + position, + eid: eid, + offset: eidOffset, + length: length, + }) + } + position, eid, eidOffset = nextPosition, nextEID, nextOffset + } + return result, nil +} + +func collectKFXContent(value any, pools map[string][]string, chunks map[int64][]kfxPositionChunk, + offsets map[int64]int64, sections *[]kfxSection, +) { + switch current := value.(type) { + case map[string]any: + if eid, ok := kfxInt(current["$155"]); ok { + if reference := kfxMap(current["$145"]); reference != nil { + poolName := kfxString(reference["name"]) + index, indexOK := kfxInt(reference["$403"]) + pool := pools[poolName] + if indexOK && index >= 0 && index < int64(len(pool)) { + text := pool[index] + offset := offsets[eid] + if position, found := kfxPositionForEID(chunks, eid, offset); found { + *sections = append(*sections, kfxSection{position: position, text: text}) + } + offsets[eid] += int64(utf8.RuneCountInString(text)) + } + } + } + for _, child := range current { + collectKFXContent(child, pools, chunks, offsets, sections) + } + case []any: + for _, child := range current { + collectKFXContent(child, pools, chunks, offsets, sections) + } + } +} + +func kfxPositionForEID(chunks map[int64][]kfxPositionChunk, eid, offset int64) (int64, bool) { + for _, chunk := range chunks[eid] { + if offset >= chunk.offset && offset < chunk.offset+chunk.length { + return chunk.position + offset - chunk.offset, true + } + } + return 0, false +} + +func collectKFXPages(value any, chunks map[int64][]kfxPositionChunk, pages *[]kfxPage) { + if kfxString(kfxMap(value)["$235"]) != "$237" { + return + } + walkKFX(value, func(_ string, child any) { + entry := kfxMap(child) + labelObject := kfxMap(entry["$241"]) + positionObject := kfxMap(entry["$246"]) + label := kfxString(labelObject["$244"]) + eid, eidOK := kfxInt(positionObject["$155"]) + offset, offsetOK := kfxInt(positionObject["$143"]) + if !offsetOK { + offset = 0 + } + if label == "" || !eidOK { + return + } + if position, found := kfxPositionForEID(chunks, eid, offset); found { + *pages = append(*pages, kfxPage{position: position, label: label}) + } + }) +} + +func walkKFX(value any, visit func(field string, value any)) { + switch v := value.(type) { + case map[string]any: + visit("", v) + for field, child := range v { + visit(field, child) + walkKFX(child, visit) + } + case []any: + visit("", v) + for _, child := range v { + walkKFX(child, visit) + } + } +} + +func (book kfxBook) textAt(start, end int64) (string, bool) { + if start > end { + start, end = end, start + } + var output strings.Builder + covered := false + for i, section := range book.sections { + sectionRunes := []rune(section.text) + sectionEnd := section.position + int64(len(sectionRunes)) + if i+1 < len(book.sections) && book.sections[i+1].position < sectionEnd { + sectionEnd = book.sections[i+1].position + } + if sectionEnd <= start || section.position > end { + continue + } + left := max(start, section.position) - section.position + right := min(end+1, sectionEnd) - section.position + if left >= 0 && right > left && right <= int64(len(sectionRunes)) { + output.WriteString(string(sectionRunes[left:right])) + covered = true + } + } + text := strings.TrimSpace(strings.Join(strings.Fields(output.String()), " ")) + return text, covered && utf8.ValidString(text) +} + +func (book kfxBook) pageAt(position int64) string { + index := sort.Search(len(book.pages), func(i int) bool { return book.pages[i].position > position }) - 1 + if index >= 0 && book.pages[index].label != "" { + return "#" + book.pages[index].label + } + return "#" + itoa64(position) +} diff --git a/internal/sdr/krds.go b/internal/sdr/krds.go index 1c13d6c..dcd5862 100644 --- a/internal/sdr/krds.go +++ b/internal/sdr/krds.go @@ -4,6 +4,8 @@ import ( "encoding/binary" "fmt" "math" + "strconv" + "strings" "time" ) @@ -370,8 +372,10 @@ func asPosition(value any) (int64, bool) { return n, true } if text, ok := value.(string); ok { - var n int64 - if _, err := fmt.Sscan(text, &n); err == nil { + if separator := strings.LastIndexByte(text, ':'); separator >= 0 { + text = text[separator+1:] + } + if n, err := strconv.ParseInt(text, 10, 64); err == nil && n >= 0 { return n, true } } diff --git a/internal/sdr/sdr_test.go b/internal/sdr/sdr_test.go index 4b9ba4f..6637bb3 100644 --- a/internal/sdr/sdr_test.go +++ b/internal/sdr/sdr_test.go @@ -8,6 +8,9 @@ import ( "strings" "testing" "time" + "unicode/utf8" + + "github.com/amazon-ion/ion-go/ion" ) func TestDecodeSidecarAndRecoverText(t *testing.T) { @@ -176,6 +179,99 @@ func TestExtractPathEndToEnd(t *testing.T) { } } +func TestExtractPathKFXEndToEnd(t *testing.T) { + dir := t.TempDir() + bookPath := filepath.Join(dir, "Example.kfx") + sidecarDir := filepath.Join(dir, "Example.sdr") + if err := os.Mkdir(sidecarDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(bookPath, buildKFXContainerFixture(t, "旅途测试", "甲乙丙丁"), 0o600); err != nil { + t.Fatal(err) + } + created := time.Date(2026, 8, 2, 9, 30, 0, 0, time.UTC) + sidecar := buildKRDSFixture(t, []fixtureAnnotation{ + {kind: 1, start: "fixture:2", end: "fixture:3", created: created}, + {kind: 2, start: "fixture:3", end: "fixture:3", created: created.Add(time.Second), note: "重点"}, + }, nil) + if err := os.WriteFile(filepath.Join(sidecarDir, "annotations.yjr"), sidecar, 0o600); err != nil { + t.Fatal(err) + } + + report, err := ExtractPath(dir) + if err != nil { + t.Fatalf("ExtractPath() error = %v; warnings = %v", err, report.Warnings) + } + if report.Decoded != 1 || len(report.Books) != 1 || len(report.Books[0].Highlights) != 1 { + t.Fatalf("unexpected KFX report: %+v", report) + } + highlight := report.Books[0].Highlights[0] + if highlight.Title != "旅途测试" || highlight.Text != "乙丙" || highlight.ExactText != "乙丙" { + t.Fatalf("unexpected KFX highlight text: %+v", highlight) + } + if highlight.Note != "重点" || highlight.PageAt != "#7" || !highlight.CreatedAt.Equal(created) { + t.Fatalf("unexpected KFX highlight metadata: %+v", highlight) + } + + direct, err := ExtractPath(bookPath) + if err != nil || direct.Decoded != 1 || len(direct.Books[0].Highlights) != 1 { + t.Fatalf("direct KFX ExtractPath() = %+v, %v", direct, err) + } +} + +func TestExtractPathKFXEmptyAnnotations(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "Empty.kfx"), buildKFXContainerFixture(t, "空标注", "没有标注"), 0o600); err != nil { + t.Fatal(err) + } + sidecarDir := filepath.Join(dir, "Empty.sdr") + if err := os.Mkdir(sidecarDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sidecarDir, "empty.yjr"), buildKRDSFixture(t, nil, nil), 0o600); err != nil { + t.Fatal(err) + } + + report, err := ExtractPath(sidecarDir) + if err != nil { + t.Fatalf("ExtractPath(empty KFX) error = %v", err) + } + if report.Decoded != 1 || len(report.Books) != 1 || len(report.Books[0].Highlights) != 0 { + t.Fatalf("unexpected empty KFX report: %+v", report) + } +} + +func TestKFXRejectsMalformedData(t *testing.T) { + valid := buildKFXContainerFixture(t, "Malformed", "text") + tests := []struct { + name string + data []byte + }{ + {name: "signature", data: []byte("NOPE")}, + {name: "truncated", data: valid[:20]}, + {name: "unsupported version", data: append([]byte(nil), valid...)}, + } + binary.LittleEndian.PutUint16(tests[2].data[4:6], 99) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := assembleKFX(test.data); err == nil { + t.Fatal("assembleKFX() accepted malformed data") + } + }) + } +} + +func TestKFXTextAtAcrossFragments(t *testing.T) { + book := kfxBook{sections: []kfxSection{ + {position: 10, text: "甲乙"}, + {position: 12, text: "丙丁"}, + }} + text, exact := book.textAt(11, 12) + if !exact || text != "乙丙" { + t.Fatalf("textAt() = %q, %v; want 乙丙, true", text, exact) + } +} + type fixtureAnnotation struct { kind int64 start string @@ -296,6 +392,104 @@ func buildKF8Fixture(t *testing.T, title string, markup []byte) []byte { return file } +type fixtureKFXEntity struct { + idSID uint32 + typeSID uint32 + data []byte +} + +func buildKFXContainerFixture(t *testing.T, title, text string) []byte { + t.Helper() + textLength := int64(utf8.RuneCountInString(text)) + entities := []fixtureKFXEntity{ + buildKFXEntityFixture(t, 900, 145, map[string]any{ + "$146": []any{text}, + }), + buildKFXEntityFixture(t, 901, 259, map[string]any{ + "$146": []any{map[string]any{ + "$145": map[string]any{"name": "$900", "$403": int64(0)}, + "$155": int64(2), "$159": "$269", + }}, + }), + buildKFXEntityFixture(t, 348, 265, map[string]any{ + "$181": []any{map[string]any{"$144": textLength + 1, "$174": "$902", "$184": int64(0)}}, + }), + buildKFXEntityFixture(t, 902, 609, map[string]any{ + "$174": "$902", + "$181": []any{[]any{int64(0), int64(1)}, []any{int64(1), int64(2)}, []any{textLength, int64(0)}}, + }), + buildKFXEntityFixture(t, 348, 490, map[string]any{ + "$491": []any{map[string]any{ + "$495": "kindle_title_metadata", + "$258": []any{map[string]any{"$492": "title", "$307": title}}, + }}, + }), + buildKFXEntityFixture(t, 903, 391, map[string]any{ + "$235": "$237", + "$247": []any{map[string]any{ + "$241": map[string]any{"$244": "7"}, + "$246": map[string]any{"$155": int64(2), "$143": int64(0)}, + }}, + }), + } + + indexLength := len(entities) * 24 + entityOffset := uint64(0) + index := make([]byte, indexLength) + for i, entity := range entities { + offset := i * 24 + binary.LittleEndian.PutUint32(index[offset:offset+4], entity.idSID) + binary.LittleEndian.PutUint32(index[offset+4:offset+8], entity.typeSID) + binary.LittleEndian.PutUint64(index[offset+8:offset+16], entityOffset) + binary.LittleEndian.PutUint64(index[offset+16:offset+24], uint64(len(entity.data))) + entityOffset += uint64(len(entity.data)) + } + containerInfo := encodeKFXIonFixture(t, map[string]any{ + "$409": "fixture", "$410": int64(0), "$411": int64(0), "$412": int64(4096), + "$413": int64(kfxContainerHeaderSize), "$414": int64(indexLength), "$416": int64(0), + }) + headerLength := kfxContainerHeaderSize + len(index) + len(containerInfo) + result := make([]byte, headerLength) + copy(result[:4], "CONT") + binary.LittleEndian.PutUint16(result[4:6], 2) + binary.LittleEndian.PutUint32(result[6:10], uint32(headerLength)) + binary.LittleEndian.PutUint32(result[10:14], uint32(kfxContainerHeaderSize+len(index))) + binary.LittleEndian.PutUint32(result[14:18], uint32(len(containerInfo))) + copy(result[kfxContainerHeaderSize:], index) + copy(result[kfxContainerHeaderSize+len(index):], containerInfo) + for _, entity := range entities { + result = append(result, entity.data...) + } + return result +} + +func buildKFXEntityFixture(t *testing.T, idSID, typeSID uint32, value any) fixtureKFXEntity { + t.Helper() + info := encodeKFXIonFixture(t, map[string]any{"$410": int64(0), "$411": int64(0)}) + payload := encodeKFXIonFixture(t, value) + headerLength := kfxEntityHeaderSize + len(info) + data := make([]byte, headerLength) + copy(data[:4], "ENTY") + binary.LittleEndian.PutUint16(data[4:6], 1) + binary.LittleEndian.PutUint32(data[6:10], uint32(headerLength)) + copy(data[kfxEntityHeaderSize:], info) + data = append(data, payload...) + return fixtureKFXEntity{idSID: idSID, typeSID: typeSID, data: data} +} + +func encodeKFXIonFixture(t *testing.T, value any) []byte { + t.Helper() + var output bytes.Buffer + w := ion.NewBinaryWriter(&output, yjSharedSymbols()) + if err := ion.MarshalTo(w, value); err != nil { + t.Fatal(err) + } + if err := w.Finish(); err != nil { + t.Fatal(err) + } + return append([]byte(nil), output.Bytes()...) +} + func buildIndexFixture(t *testing.T, tags []indexTag, text string, control byte, values []uint64) ([]byte, []byte) { t.Helper() tagLength := 12 + len(tags)*4 @@ -336,7 +530,7 @@ func buildIndexFixture(t *testing.T, tags []indexTag, text string, control byte, return main, extra } -func TestDiscoverSkipsKFXWithWarning(t *testing.T) { +func TestDiscoverWarnsForMissingKFXBook(t *testing.T) { dir := t.TempDir() sidecar := filepath.Join(dir, "KFX.sdr") if err := os.Mkdir(sidecar, 0o755); err != nil { diff --git a/internal/sdr/types.go b/internal/sdr/types.go index bd053b4..ca0cd9c 100644 --- a/internal/sdr/types.go +++ b/internal/sdr/types.go @@ -26,7 +26,7 @@ type PageMap struct { Positions []int64 } -// Sidecar is the annotation data decoded from one or more .azw3r files. +// Sidecar is annotation data decoded from Kindle reader-data-store files. type Sidecar struct { Annotations []Annotation PageMap PageMap From 619f4c124075a519d9cf737c198fecfd37971f11 Mon Sep 17 00:00:00 2001 From: AnnatarHe Date: Sun, 2 Aug 2026 10:10:18 +0800 Subject: [PATCH 3/3] fix(cli): handle empty Kindle annotation caches --- internal/sdr/extract.go | 125 +++++++++++++++++++++++++++++---------- internal/sdr/sdr_test.go | 42 ++++++++++++- 2 files changed, 135 insertions(+), 32 deletions(-) diff --git a/internal/sdr/extract.go b/internal/sdr/extract.go index 474f776..0ef98ab 100644 --- a/internal/sdr/extract.go +++ b/internal/sdr/extract.go @@ -13,6 +13,11 @@ type bookCandidate struct { sidecarDir string } +type extractionStats struct { + annotationCount int + warnings []string +} + // ExtractPath discovers Kindle book/sidecar pairs beneath path and extracts // all supported text annotations. It never writes to the source tree. func ExtractPath(path string) (Report, error) { @@ -30,8 +35,9 @@ func ExtractPath(path string) (Report, error) { return Report{}, err } report := Report{Warnings: discoveryWarnings} + extracted := 0 for _, candidate := range candidates { - result, err := extractCandidate(candidate) + result, stats, err := extractCandidate(candidate) if err != nil { if direct { return Report{}, err @@ -41,6 +47,21 @@ func ExtractPath(path string) (Report, error) { } report.Decoded++ report.Books = append(report.Books, result) + report.Warnings = append(report.Warnings, stats.warnings...) + extracted += len(result.Highlights) + if stats.annotationCount == 0 { + sidecarType := ".azw3r" + if strings.EqualFold(filepath.Ext(candidate.bookPath), ".kfx") { + sidecarType = ".yjr" + } + report.Warnings = append(report.Warnings, fmt.Sprintf( + "%s: annotation cache is empty; no local highlight positions exist in the %s files (sync and open the book on the Kindle before copying it, or use documents/My Clippings.txt)", + candidate.sidecarDir, sidecarType)) + } else if len(result.Highlights) == 0 { + report.Warnings = append(report.Warnings, fmt.Sprintf( + "%s: parsed %d text annotations, but none could be resolved against %s", + candidate.sidecarDir, stats.annotationCount, filepath.Base(candidate.bookPath))) + } } if report.Decoded == 0 { if len(report.Warnings) > 0 { @@ -48,6 +69,9 @@ func ExtractPath(path string) (Report, error) { } return report, fmt.Errorf("no supported Kindle sidecar pairs found under %s", path) } + if extracted == 0 { + return report, fmt.Errorf("no highlighted text could be extracted from %s", path) + } return report, nil } @@ -158,6 +182,10 @@ func sidecarFormats(directory string) (hasAZW3R, hasYJR bool) { hasAZW3R = true case ".yjr": hasYJR = true + default: + if strings.HasSuffix(strings.ToLower(entry.Name()), ".yjr.bad_file") { + hasYJR = true + } } } return hasAZW3R, hasYJR @@ -172,40 +200,67 @@ func supportedBookExtension(extension string) bool { } } -func extractCandidate(candidate bookCandidate) (BookResult, error) { +func extractCandidate(candidate bookCandidate) (BookResult, extractionStats, error) { bookExtension := strings.ToLower(filepath.Ext(candidate.bookPath)) - sidecarPaths, err := findSidecarFiles(candidate.sidecarDir, bookExtension) + sidecarPaths, fallbackPaths, err := findSidecarFiles(candidate.sidecarDir, bookExtension) if err != nil { - return BookResult{}, err + return BookResult{}, extractionStats{}, err } merged := Sidecar{} seenAnnotations := make(map[string]struct{}) - for _, path := range sidecarPaths { - data, err := os.ReadFile(path) - if err != nil { - return BookResult{}, fmt.Errorf("read %s: %w", filepath.Base(path), err) - } - decoded, err := DecodeSidecar(data) - if err != nil { - return BookResult{}, fmt.Errorf("decode %s: %w", filepath.Base(path), err) - } - if len(merged.PageMap.Positions) == 0 && len(decoded.PageMap.Positions) > 0 { - merged.PageMap = decoded.PageMap - } - for _, annotation := range decoded.Annotations { - key := fmt.Sprintf("%s\x00%d\x00%d\x00%d\x00%s", annotation.Type, - annotation.StartPosition, annotation.EndPosition, annotation.CreationTime.UnixMilli(), annotation.Note) - if _, duplicate := seenAnnotations[key]; duplicate { + stats := extractionStats{} + decodePaths := func(paths []string, required bool) error { + before := len(merged.Annotations) + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + if required { + return fmt.Errorf("read %s: %w", filepath.Base(path), err) + } + stats.warnings = append(stats.warnings, fmt.Sprintf("%s: cannot read fallback %s: %v", candidate.sidecarDir, filepath.Base(path), err)) continue } - seenAnnotations[key] = struct{}{} - merged.Annotations = append(merged.Annotations, annotation) + decoded, err := DecodeSidecar(data) + if err != nil { + if required { + return fmt.Errorf("decode %s: %w", filepath.Base(path), err) + } + stats.warnings = append(stats.warnings, fmt.Sprintf("%s: cannot decode fallback %s: %v", candidate.sidecarDir, filepath.Base(path), err)) + continue + } + if len(merged.PageMap.Positions) == 0 && len(decoded.PageMap.Positions) > 0 { + merged.PageMap = decoded.PageMap + } + for _, annotation := range decoded.Annotations { + key := fmt.Sprintf("%s\x00%d\x00%d\x00%d\x00%s", annotation.Type, + annotation.StartPosition, annotation.EndPosition, annotation.CreationTime.UnixMilli(), annotation.Note) + if _, duplicate := seenAnnotations[key]; duplicate { + continue + } + seenAnnotations[key] = struct{}{} + merged.Annotations = append(merged.Annotations, annotation) + } } + if !required && len(merged.Annotations) > before { + stats.warnings = append(stats.warnings, fmt.Sprintf( + "%s: active .yjr cache was empty; recovered annotations from .yjr.bad_file", + candidate.sidecarDir)) + } + return nil + } + if err := decodePaths(sidecarPaths, true); err != nil { + return BookResult{}, stats, err } + if bookExtension == ".kfx" && len(merged.Annotations) == 0 && len(fallbackPaths) > 0 { + if err := decodePaths(fallbackPaths, false); err != nil { + return BookResult{}, stats, err + } + } + stats.annotationCount = len(merged.Annotations) bookData, err := os.ReadFile(candidate.bookPath) if err != nil { - return BookResult{}, fmt.Errorf("read book: %w", err) + return BookResult{}, stats, fmt.Errorf("read book: %w", err) } var title string var assembled []byte @@ -213,13 +268,13 @@ func extractCandidate(candidate bookCandidate) (BookResult, error) { if bookExtension == ".kfx" { kfx, err = assembleKFX(bookData) if err != nil { - return BookResult{}, fmt.Errorf("assemble %s: %w", filepath.Base(candidate.bookPath), err) + return BookResult{}, stats, fmt.Errorf("assemble %s: %w", filepath.Base(candidate.bookPath), err) } title = kfx.title } else { title, assembled, err = AssembleBook(bookData) if err != nil { - return BookResult{}, fmt.Errorf("assemble %s: %w", filepath.Base(candidate.bookPath), err) + return BookResult{}, stats, fmt.Errorf("assemble %s: %w", filepath.Base(candidate.bookPath), err) } } if title == "" { @@ -264,7 +319,7 @@ func extractCandidate(candidate bookCandidate) (BookResult, error) { PageAt: page, }) } - return result, nil + return result, stats, nil } func associateNotes(annotations []Annotation) []Annotation { @@ -298,12 +353,13 @@ func associateNotes(annotations []Annotation) []Annotation { return filtered } -func findSidecarFiles(directory, bookExtension string) ([]string, error) { +func findSidecarFiles(directory, bookExtension string) ([]string, []string, error) { entries, err := os.ReadDir(directory) if err != nil { - return nil, fmt.Errorf("read sidecar directory: %w", err) + return nil, nil, fmt.Errorf("read sidecar directory: %w", err) } var paths []string + var fallbacks []string wanted := ".azw3r" if bookExtension == ".kfx" { wanted = ".yjr" @@ -312,13 +368,20 @@ func findSidecarFiles(directory, bookExtension string) ([]string, error) { if entry.IsDir() { continue } - if strings.ToLower(filepath.Ext(entry.Name())) == wanted { + lowerName := strings.ToLower(entry.Name()) + if strings.ToLower(filepath.Ext(lowerName)) == wanted { paths = append(paths, filepath.Join(directory, entry.Name())) + } else if bookExtension == ".kfx" && strings.HasSuffix(lowerName, ".yjr.bad_file") { + fallbacks = append(fallbacks, filepath.Join(directory, entry.Name())) } } + if len(paths) == 0 && len(fallbacks) == 0 { + return nil, nil, fmt.Errorf("sidecar contains no %s file", wanted) + } if len(paths) == 0 { - return nil, fmt.Errorf("sidecar contains no %s file", wanted) + paths, fallbacks = fallbacks, nil } sort.Strings(paths) - return paths, nil + sort.Strings(fallbacks) + return paths, fallbacks, nil } diff --git a/internal/sdr/sdr_test.go b/internal/sdr/sdr_test.go index 6637bb3..39384d4 100644 --- a/internal/sdr/sdr_test.go +++ b/internal/sdr/sdr_test.go @@ -233,12 +233,52 @@ func TestExtractPathKFXEmptyAnnotations(t *testing.T) { } report, err := ExtractPath(sidecarDir) - if err != nil { + if err == nil || !strings.Contains(err.Error(), "no highlighted text") { t.Fatalf("ExtractPath(empty KFX) error = %v", err) } if report.Decoded != 1 || len(report.Books) != 1 || len(report.Books[0].Highlights) != 0 { t.Fatalf("unexpected empty KFX report: %+v", report) } + if len(report.Warnings) != 1 || !strings.Contains(report.Warnings[0], "annotation cache is empty") { + t.Fatalf("empty KFX warnings = %v", report.Warnings) + } +} + +func TestExtractPathKFXRecoversBadFileFallback(t *testing.T) { + dir := t.TempDir() + bookPath := filepath.Join(dir, "Recovered.kfx") + if err := os.WriteFile(bookPath, buildKFXContainerFixture(t, "恢复测试", "甲乙丙丁"), 0o600); err != nil { + t.Fatal(err) + } + sidecarDir := filepath.Join(dir, "Recovered.sdr") + if err := os.Mkdir(sidecarDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sidecarDir, "active.yjr"), buildKRDSFixture(t, nil, nil), 0o600); err != nil { + t.Fatal(err) + } + created := time.Date(2026, 8, 2, 3, 4, 5, 0, time.UTC) + backup := buildKRDSFixture(t, []fixtureAnnotation{ + {kind: 1, start: "AVgCAAAoAAAA:1", end: "AVgCAAAoAAAA:2", created: created}, + }, nil) + if err := os.WriteFile(filepath.Join(sidecarDir, "backup.yjr.bad_file"), backup, 0o600); err != nil { + t.Fatal(err) + } + + report, err := ExtractPath(sidecarDir) + if err != nil { + t.Fatalf("ExtractPath(fallback KFX) error = %v; warnings = %v", err, report.Warnings) + } + if report.Decoded != 1 || len(report.Books) != 1 || len(report.Books[0].Highlights) != 1 { + t.Fatalf("unexpected fallback KFX report: %+v", report) + } + highlight := report.Books[0].Highlights[0] + if highlight.Text != "甲乙" || !highlight.CreatedAt.Equal(created) { + t.Fatalf("fallback highlight = %+v", highlight) + } + if len(report.Warnings) != 1 || !strings.Contains(report.Warnings[0], "recovered annotations") { + t.Fatalf("fallback warnings = %v", report.Warnings) + } } func TestKFXRejectsMalformedData(t *testing.T) {