Crosspoint/docs/file-formats.md

File Formats

book.bin

Version 7

ImHex Pattern:

import std.mem;
import std.string;
import std.core;

// === Configuration ===
#define EXPECTED_VERSION 7
#define MAX_STRING_LENGTH 65535

// === String Structure ===

struct String {
    u32 length [[hidden, comment("String byte length")]];
    if (length > MAX_STRING_LENGTH) {
        std::warning(std::format("Unusually large string length: {} bytes", length));
    }
    char data[length] [[comment("UTF-8 string data")]];
} [[sealed, format("format_string"), comment("Length-prefixed UTF-8 string")]];

fn format_string(String s) {
    return s.data;
};

// === Metadata Structure ===

struct Metadata {
    String title [[comment("Book title")]];
    String author [[comment("Book author")]];
    String language [[comment("BCP47 language tag")]];
    String coverItemHref [[comment("Path to cover image")]];
    String textReferenceHref [[comment("Path to guided first text reference")]];
    String series [[comment("Series name")]];
    String seriesIndex [[comment("Series index/position")]];
    String description [[comment("Book description / blurb")]];
} [[comment("Book metadata information")]];

// === Spine Entry Structure ===

struct SpineEntry {
    String href [[comment("Resource path")]];
    u32 cumulativeSize [[comment("Cumulative size in bytes"), color("FF6B6B")]];
    s16 tocIndex [[comment("Index into TOC (-1 if none)"), color("4ECDC4")]];
} [[comment("Spine entry defining reading order")]];

// === TOC Entry Structure ===

struct TocEntry {
    String title [[comment("Chapter/section title")]];
    String href [[comment("Resource path")]];
    String anchor [[comment("Fragment identifier")]];
    u8 level [[comment("Nesting level (0-255)"), color("95E1D3")]];
    s16 spineIndex [[comment("Index into spine (-1 if none)"), color("F38181")]];
} [[comment("Table of contents entry")]];

// === Book Bin Structure ===

struct BookBin {
    // Header
    u8 version [[comment("Format version"), color("FFD93D")]];
    
    // Version validation
    if (version != EXPECTED_VERSION) {
        std::error(std::format("Unsupported version: {} (expected {})", version, EXPECTED_VERSION));
    }
    
    u32 lutOffset [[comment("Offset to lookup tables"), color("6BCB77")]];
    u16 spineCount [[comment("Number of spine entries"), color("4D96FF")]];
    u16 tocCount [[comment("Number of TOC entries"), color("FF6B9D")]];
    u8 tocReliable [[comment("1 if TOC has >=25% spine coverage, 0 otherwise"), color("F4A261")]];

    // Metadata section
    Metadata metadata [[comment("Book metadata")]];
    
    // Validate LUT offset alignment
    u32 currentOffset = $;
    if (currentOffset != lutOffset) {
        std::warning(std::format("LUT offset mismatch: expected 0x{:X}, got 0x{:X}", lutOffset, currentOffset));
    }
    
    // Lookup Tables
    u32 spineLut[spineCount] [[comment("Spine entry offsets"), color("4D96FF")]];
    u32 tocLut[tocCount] [[comment("TOC entry offsets"), color("FF6B9D")]];
    
    // Data Entries
    SpineEntry spines[spineCount] [[comment("Spine entries (reading order)")]];
    TocEntry toc[tocCount] [[comment("Table of contents entries")]];
};

// === File Parsing ===

BookBin book @ 0x00;

// Validate we've consumed the entire file
u32 fileSize = std::mem::size();
u32 parsedSize = $;

if (parsedSize != fileSize) {
    std::warning(std::format("Unparsed data detected: {} bytes remaining at offset 0x{:X}", fileSize - parsedSize, parsedSize));
}

section.bin

Current version

SECTION_FILE_VERSION = 28. The on-disk layout has evolved past the v21 pattern shown below; the ImHex pattern is preserved for archeology but no longer reflects all fields. Changes since v21 (read lib/Epub/Epub/Section.cpp header::* constants for the authoritative layout):

• parseComplete (bool) inserted before pageCount so a truncated parse can be detected on reload.
• paragraphLutOffset extended: each per-page entry is now u32 xhtmlByteOffset + u16 paragraphIndex + u16 listItemIndex (added the running <li> count for KOReader list-item XPath sync).
• pageBreakMapOffset (u32) added in the header between anchorMapOffset and paragraphLutOffset. The block at that offset stores printed-page labels: u16 count, then per entry u16 pageIndex + String label. Populated from inline doc-pagebreak markers and from the per-book pagelist.bin (NCX <pageList> / EPUB 3 <nav epub:type="page-list"> / EPUB 2.01 page-map.xml). See docs/epub-toc-navigation.md for the source-format selection rules.

Version 21

ImHex Pattern:

import std.mem;
import std.string;
import std.core;

// === Configuration ===
#define EXPECTED_VERSION 21
#define MAX_STRING_LENGTH 65535

// === String Structure ===

struct String {
    u32 length [[hidden, comment("String byte length")]];
    if (length > MAX_STRING_LENGTH) {
        std::warning(std::format("Unusually large string length: {} bytes", length));
    }
    char data[length] [[comment("UTF-8 string data")]];
} [[sealed, format("format_string"), comment("Length-prefixed UTF-8 string")]];

fn format_string(String s) {
    return s.data;
};

// === Page Structure ===

enum StorageType : u8 {
    PageLine = 1
};

enum WordStyle : u8 {
    REGULAR = 0,
    BOLD = 1,
    ITALIC = 2,
    BOLD_ITALIC = 3
};

enum BlockStyle : u8 {
    JUSTIFIED = 0,
    LEFT_ALIGN = 1,
    CENTER_ALIGN = 2,
    RIGHT_ALIGN = 3,
};

struct PageLine {
  s16 xPos;
  s16 yPos;
  u16 wordCount;
  String words[wordCount];
  u16 wordXPos[wordCount];
  WordStyle wordStyle[wordCount];
  BlockStyle blockStyle;
};

struct PageElement {
    u8 pageElementType;
    if (pageElementType == 1) {
        PageLine pageLine [[inline]];
    } else {
        std::error(std::format("Unknown page element type: {}", pageElementType));
    }
};

struct Page {
    u16 elementCount;
    PageElement elements[elementCount] [[inline]];
};

// === Anchor Map Entry ===

struct AnchorEntry {
    String anchorId [[comment("HTML id attribute value")]];
    u16 pageNumber [[comment("Page where the anchor appears")]];
};

// === Section Bin Structure ===

struct SectionBin {
    // Header
    u8 version [[comment("Format version"), color("FFD93D")]];

    // Version validation
    if (version != EXPECTED_VERSION) {
        std::error(std::format("Unsupported version: {} (expected {})", version, EXPECTED_VERSION));
    }

    // Cache busting parameters
    s32 fontId;
    float lineCompression;
    bool extraParagraphSpacing;
    u8 paragraphAlignment;
    u16 viewportWidth;
    u16 viewportHeight;
    u16 pageCount;
    bool hyphenationEnabled;
    bool embeddedStyle;
    u8 imageRendering;
    u32 pageLutOffset [[comment("Offset to page offset LUT")]];
    u32 anchorMapOffset [[comment("Offset to anchor map")]];
    u32 paragraphLutOffset [[comment("Offset to per-page paragraph LUT (byte offset + <p> index)")]];

    Page page[pageCount];

    // === Page Offset LUT ===
    // Validate LUT offset alignment
    u32 currentOffset = $;
    if (currentOffset != pageLutOffset) {
        std::warning(std::format("Page LUT offset mismatch: expected 0x{:X}, got 0x{:X}", pageLutOffset, currentOffset));
    }

    u32 pageOffsets[pageCount] [[comment("File offsets to serialized pages")]];

    // === Anchor Map ===
    u16 anchorCount;
    AnchorEntry anchors[anchorCount];

    // === Paragraph LUT (deep entries) ===
    // One entry per page: XHTML byte offset at the page break + 1-based <p> sibling index.
    // xhtmlByteOffset is the Expat byte position within the decompressed spine XHTML at the
    // moment the page break fired — used as a seek hint to avoid scanning from byte 0 when
    // generating XPaths for upload.  0 means no hint (last page, recorded post-parse).
    // paragraphIndex is 1-based, matching KOReader XPath p[N] convention.
    struct ParagraphLutEntry { u32 xhtmlByteOffset; u16 paragraphIndex; };
    u16 paragraphEntryCount;
    ParagraphLutEntry paragraphLut[paragraphEntryCount];
};

// === File Parsing ===

SectionBin book @ 0x00;

// Validate we've consumed the entire file
u32 fileSize = std::mem::size();
u32 parsedSize = $;

if (parsedSize != fileSize) {
    std::warning(std::format("Unparsed data detected: {} bytes remaining at offset 0x{:X}", fileSize - parsedSize, parsedSize));
}

pagelist.bin

Per-book cache file produced at index time from one of the EPUB printed-page sources (NCX <pageList>, EPUB 3 nav <nav epub:type="page-list">, or EPUB 2.01 page-map.xml). Consumed once per section build by Section::createSectionFile. Absent for books that have no printed-page data.

u16 entryCount
struct PageListEntry {
    String href;    // normalised spine href, e.g. "OEBPS/c9_split_000.xhtml"
    String anchor;  // fragment id; empty means "start of file"
    String label;   // printed-page label, e.g. "42" or "iv"
}
PageListEntry entries[entryCount];

Selection rules (see docs/epub-toc-navigation.md):

• The EPUB 3 nav page-list parser runs first.
• The NCX <pageList> writer runs only if the nav writer produced nothing.
• The EPUB 2.01 page-map.xml writer runs only if pagelist.bin doesn't already exist on disk.
• Inline doc-pagebreak markers in XHTML are matched at chapter parse time and don't need the cache file; they coexist with whichever source above won.