Extract book cover utilities into BookCoverUtils

Refactors book cover and metadata handling out of Epub class into a dedicated BookCoverUtils utility. Moves FootnoteEntry struct into reader activity directory. Simplifies cache clearing to use direct storage operations instead of Epub class.

test/CMakeLists.txt

@@ -41,7 +41,6 @@ include(GoogleTest)
 add_subdirectory(streaming_json_parser)
 add_subdirectory(release_json_parser)
 add_subdirectory(differential_rounding)
-add_subdirectory(hyphenation_eval)
 add_subdirectory(utf8_compose)
 add_subdirectory(cpfont_adapter)
 add_subdirectory(book_xpath)

test/hyphenation_eval/CMakeLists.txt

@@ -1,24 +0,0 @@
-add_executable(HyphenationEvaluationTest
-  HyphenationEvaluationTest.cpp
-  ${REPO_ROOT}/lib/Epub/Epub/hyphenation/Hyphenator.cpp
-  ${REPO_ROOT}/lib/Epub/Epub/hyphenation/LanguageRegistry.cpp
-  ${REPO_ROOT}/lib/Epub/Epub/hyphenation/LiangHyphenation.cpp
-  ${REPO_ROOT}/lib/Epub/Epub/hyphenation/HyphenationCommon.cpp
-  ${REPO_ROOT}/lib/Utf8/Utf8.cpp
-)
-
-target_include_directories(HyphenationEvaluationTest PRIVATE
-  ${REPO_ROOT}/lib/Epub
-  ${REPO_ROOT}/lib/Utf8
-)
-
-target_compile_definitions(HyphenationEvaluationTest PRIVATE
-  HYPHENATION_RESOURCES_DIR="${CMAKE_CURRENT_SOURCE_DIR}/resources"
-)
-
-target_link_libraries(HyphenationEvaluationTest PRIVATE
-  crosspoint_test_common
-  GTest::gtest_main
-)
-
-gtest_discover_tests(HyphenationEvaluationTest)

test/hyphenation_eval/HyphenationEvaluationTest.cpp

@@ -1,234 +0,0 @@
-#include <Utf8.h>
-#include <gtest/gtest.h>
-
-#include <algorithm>
-#include <fstream>
-#include <iostream>
-#include <sstream>
-#include <string>
-#include <vector>
-
-#include "lib/Epub/Epub/hyphenation/HyphenationCommon.h"
-#include "lib/Epub/Epub/hyphenation/LanguageHyphenator.h"
-#include "lib/Epub/Epub/hyphenation/LanguageRegistry.h"
-
-#ifndef HYPHENATION_RESOURCES_DIR
-#error "HYPHENATION_RESOURCES_DIR must be defined by the build system"
-#endif
-
-namespace {
-
-struct TestCase {
-  std::string word;
-  std::string hyphenated;
-  std::vector<size_t> expectedPositions;
-  int frequency;
-};
-
-struct EvaluationResult {
-  int truePositives = 0;
-  int falsePositives = 0;
-  int falseNegatives = 0;
-  double precision = 0.0;
-  double recall = 0.0;
-  double f1Score = 0.0;
-  double weightedScore = 0.0;
-};
-
-std::vector<size_t> expectedPositionsFromAnnotatedWord(const std::string& annotated) {
-  std::vector<size_t> positions;
-  const unsigned char* ptr = reinterpret_cast<const unsigned char*>(annotated.c_str());
-  size_t codepointIndex = 0;
-
-  while (*ptr != 0) {
-    if (*ptr == '=') {
-      positions.push_back(codepointIndex);
-      ++ptr;
-      continue;
-    }
-
-    utf8NextCodepoint(&ptr);
-    ++codepointIndex;
-  }
-
-  return positions;
-}
-
-std::vector<TestCase> loadTestData(const std::string& filename) {
-  std::vector<TestCase> testCases;
-  std::ifstream file(filename);
-
-  if (!file.is_open()) {
-    return testCases;
-  }
-
-  std::string line;
-  while (std::getline(file, line)) {
-    if (line.empty() || line[0] == '#') {
-      continue;
-    }
-
-    std::istringstream iss(line);
-    std::string word, hyphenated, freqStr;
-
-    if (std::getline(iss, word, '|') && std::getline(iss, hyphenated, '|') && std::getline(iss, freqStr, '|')) {
-      TestCase testCase;
-      testCase.word = word;
-      testCase.hyphenated = hyphenated;
-      testCase.frequency = std::stoi(freqStr);
-      testCase.expectedPositions = expectedPositionsFromAnnotatedWord(hyphenated);
-      testCases.push_back(testCase);
-    }
-  }
-
-  return testCases;
-}
-
-std::string positionsToHyphenated(const std::string& word, const std::vector<size_t>& positions) {
-  std::string result;
-  std::vector<size_t> sortedPositions = positions;
-  std::sort(sortedPositions.begin(), sortedPositions.end());
-
-  const unsigned char* ptr = reinterpret_cast<const unsigned char*>(word.c_str());
-  size_t codepointIndex = 0;
-  size_t posIdx = 0;
-
-  while (*ptr != 0) {
-    while (posIdx < sortedPositions.size() && sortedPositions[posIdx] == codepointIndex) {
-      result.push_back('=');
-      ++posIdx;
-    }
-
-    const unsigned char* current = ptr;
-    utf8NextCodepoint(&ptr);
-    result.append(reinterpret_cast<const char*>(current), reinterpret_cast<const char*>(ptr));
-    ++codepointIndex;
-  }
-
-  while (posIdx < sortedPositions.size() && sortedPositions[posIdx] == codepointIndex) {
-    result.push_back('=');
-    ++posIdx;
-  }
-
-  return result;
-}
-
-std::vector<size_t> hyphenateWordWithHyphenator(const std::string& word, const LanguageHyphenator& hyphenator) {
-  auto cps = collectCodepoints(word);
-  trimSurroundingPunctuationAndFootnote(cps);
-  return hyphenator.breakIndexes(cps);
-}
-
-EvaluationResult evaluateWord(const TestCase& testCase, const std::vector<size_t>& actualPositions) {
-  EvaluationResult result;
-
-  std::vector<size_t> expected = testCase.expectedPositions;
-  std::vector<size_t> actual = actualPositions;
-
-  std::sort(expected.begin(), expected.end());
-  std::sort(actual.begin(), actual.end());
-
-  for (size_t pos : actual) {
-    if (std::find(expected.begin(), expected.end(), pos) != expected.end()) {
-      result.truePositives++;
-    } else {
-      result.falsePositives++;
-    }
-  }
-
-  for (size_t pos : expected) {
-    if (std::find(actual.begin(), actual.end(), pos) == actual.end()) {
-      result.falseNegatives++;
-    }
-  }
-
-  if (result.truePositives + result.falsePositives > 0) {
-    result.precision = static_cast<double>(result.truePositives) / (result.truePositives + result.falsePositives);
-  }
-
-  if (result.truePositives + result.falseNegatives > 0) {
-    result.recall = static_cast<double>(result.truePositives) / (result.truePositives + result.falseNegatives);
-  }
-
-  if (result.precision + result.recall > 0) {
-    result.f1Score = 2 * result.precision * result.recall / (result.precision + result.recall);
-  }
-
-  // Treat words with no expected and no actual hyphenation marks as perfect.
-  if (expected.empty() && actual.empty()) {
-    result.precision = 1.0;
-    result.recall = 1.0;
-    result.f1Score = 1.0;
-  }
-
-  double fpPenalty = 2.0;
-  double fnPenalty = 1.0;
-  int totalErrors = result.falsePositives * fpPenalty + result.falseNegatives * fnPenalty;
-  int totalPossible = static_cast<int>(expected.size() * fpPenalty);
-
-  if (totalPossible > 0) {
-    result.weightedScore = 1.0 - (static_cast<double>(totalErrors) / totalPossible);
-    result.weightedScore = std::max(0.0, result.weightedScore);
-  } else if (result.falsePositives == 0) {
-    result.weightedScore = 1.0;
-  }
-
-  return result;
-}
-
-// Runs the evaluation for a single language and asserts the per-word average F1
-// is at or above `minF1Percent`. Thresholds are set ~1pp below measured
-// baselines so unrelated tweaks don't fail CI but real regressions still trip.
-void runLanguageEval(const char* langName, const char* primaryTag, const char* resourceFile, double minF1Percent) {
-  const auto* hyphenator = getLanguageHyphenatorForPrimaryTag(primaryTag);
-  ASSERT_NE(hyphenator, nullptr) << "No hyphenator registered for tag: " << primaryTag;
-
-  std::string path = std::string(HYPHENATION_RESOURCES_DIR) + "/" + resourceFile;
-  std::vector<TestCase> testCases = loadTestData(path);
-  ASSERT_FALSE(testCases.empty()) << "No test cases loaded from " << path;
-
-  double totalF1 = 0.0;
-  std::vector<std::pair<TestCase, EvaluationResult>> imperfect;
-
-  for (const auto& tc : testCases) {
-    std::vector<size_t> actual = hyphenateWordWithHyphenator(tc.word, *hyphenator);
-    EvaluationResult res = evaluateWord(tc, actual);
-    totalF1 += res.f1Score;
-    if (res.weightedScore < 0.999999) {
-      imperfect.emplace_back(tc, res);
-    }
-  }
-
-  double averageF1Percent = totalF1 / testCases.size() * 100.0;
-  ::testing::Test::RecordProperty("avg_f1_percent", std::to_string(averageF1Percent));
-  ::testing::Test::RecordProperty("test_cases", std::to_string(testCases.size()));
-
-  std::cout << langName << ": F1=" << averageF1Percent << "% (threshold " << minF1Percent << "%, " << testCases.size()
-            << " cases)\n";
-
-  if (averageF1Percent < minF1Percent) {
-    std::sort(imperfect.begin(), imperfect.end(),
-              [](const auto& a, const auto& b) { return a.second.weightedScore < b.second.weightedScore; });
-    std::cout << "Worst cases for " << langName << ":\n";
-    int show = std::min<int>(10, static_cast<int>(imperfect.size()));
-    for (int i = 0; i < show; ++i) {
-      const TestCase& tc = imperfect[i].first;
-      std::vector<size_t> actual = hyphenateWordWithHyphenator(tc.word, *hyphenator);
-      std::cout << "  " << tc.word << " | expected=" << tc.hyphenated
-                << " | got=" << positionsToHyphenated(tc.word, actual) << "\n";
-    }
-  }
-
-  EXPECT_GE(averageF1Percent, minF1Percent) << "Hyphenation quality regressed for " << langName;
-}
-
-}  // namespace
-
-TEST(HyphenationEval, English) { runLanguageEval("english", "en", "english_hyphenation_tests.txt", 98.10); }
-TEST(HyphenationEval, French) { runLanguageEval("french", "fr", "french_hyphenation_tests.txt", 99.00); }
-TEST(HyphenationEval, German) { runLanguageEval("german", "de", "german_hyphenation_tests.txt", 96.73); }
-TEST(HyphenationEval, Russian) { runLanguageEval("russian", "ru", "russian_hyphenation_tests.txt", 96.22); }
-TEST(HyphenationEval, Spanish) { runLanguageEval("spanish", "es", "spanish_hyphenation_tests.txt", 98.02); }
-TEST(HyphenationEval, Italian) { runLanguageEval("italian", "it", "italian_hyphenation_tests.txt", 98.99); }
-TEST(HyphenationEval, Polish) { runLanguageEval("polish", "pl", "polish_hyphenation_tests.txt", 98.92); }
-TEST(HyphenationEval, Swedish) { runLanguageEval("swedish", "sv", "swedish_hyphenation_tests.txt", 94.01); }

test/hyphenation_eval/resources/generate_hyphenation_test_data.py

@@ -1,233 +0,0 @@
-"""
-Generate hyphenation test data from a text file.
-
-This script extracts unique words from a book and generates ground truth
-hyphenations using the pyphen library, which can be used to test and validate
-the hyphenation implementations (e.g., German, English, Russian).
-
-Usage:
-    python generate_hyphenation_test_data.py <input_file> <output_file>
-        [--language de_DE] [--max-words 5000] [--min-prefix 2] [--min-suffix 2]
-
-Requirements:
-    pip install pyphen
-"""
-
-import argparse
-import re
-from collections import Counter
-from pathlib import Path
-import zipfile
-
-
-def extract_text_from_epub(epub_path):
-    """Extract textual content from an .epub archive by concatenating HTML/XHTML files."""
-    texts = []
-    with zipfile.ZipFile(epub_path, "r") as z:
-        for name in z.namelist():
-            lower = name.lower()
-            if (
-                lower.endswith(".xhtml")
-                or lower.endswith(".html")
-                or lower.endswith(".htm")
-            ):
-                try:
-                    data = z.read(name).decode("utf-8", errors="ignore")
-                except Exception:
-                    continue
-                # Remove tags
-                text = re.sub(r"<[^>]+>", " ", data)
-                texts.append(text)
-    return "\n".join(texts)
-
-
-def extract_words(text):
-    """Extract all words from text, preserving original case."""
-    # Match runs of Unicode letters (any script) while excluding digits/underscores
-    return re.findall(r"[^\W\d_]+", text, flags=re.UNICODE)
-
-
-def clean_word(word):
-    """Normalize word for hyphenation testing."""
-    # Keep original case but strip any non-letter characters
-    return word.strip()
-
-
-def generate_hyphenation_data(
-    input_file,
-    output_file,
-    language="de_DE",
-    min_length=6,
-    max_words=5000,
-    min_prefix=2,
-    min_suffix=2,
-):
-    """
-    Generate hyphenation test data from a text file.
-
-    Args:
-        input_file: Path to input text file
-        output_file: Path to output file with hyphenation data
-        language: Language code for pyphen (e.g., 'de_DE', 'en_US')
-        min_length: Minimum word length to include
-        max_words: Maximum number of words to include (default: 5000)
-        min_prefix: Minimum characters allowed before the first hyphen (default: 2)
-        min_suffix: Minimum characters allowed after the last hyphen (default: 2)
-    """
-    import pyphen
-
-    print(f"Reading from: {input_file}")
-
-    # Read the input file
-    if str(input_file).lower().endswith(".epub"):
-        print("Detected .epub input; extracting HTML content")
-        text = extract_text_from_epub(input_file)
-    else:
-        with open(input_file, "r", encoding="utf-8") as f:
-            text = f.read()
-
-    # Extract words
-    print("Extracting words...")
-    words = extract_words(text)
-    print(f"Found {len(words)} total words")
-
-    # Count word frequencies
-    word_counts = Counter(words)
-    print(f"Found {len(word_counts)} unique words")
-
-    # Initialize pyphen hyphenator
-    print(
-        f"Initializing hyphenator for language: {language} (min_prefix={min_prefix}, min_suffix={min_suffix})"
-    )
-    try:
-        hyphenator = pyphen.Pyphen(lang=language, left=min_prefix, right=min_suffix)
-    except KeyError:
-        print(f"Error: Language '{language}' not found in pyphen.")
-        print("Available languages include: de_DE, en_US, en_GB, fr_FR, etc.")
-        return
-
-    # Generate hyphenations
-    print("Generating hyphenations...")
-    hyphenation_data = []
-
-    # Sort by frequency (most common first) then alphabetically
-    sorted_words = sorted(word_counts.items(), key=lambda x: (-x[1], x[0].lower()))
-
-    for word, count in sorted_words:
-        # Filter by minimum length
-        if len(word) < min_length:
-            continue
-
-        # Get hyphenation (may produce no '=' characters)
-        hyphenated = hyphenator.inserted(word, hyphen="=")
-
-        # Include all words (so we can take the top N most common words even if
-        # they don't have hyphenation points). This replaces the previous filter
-        # which dropped words without '='.
-        hyphenation_data.append(
-            {"word": word, "hyphenated": hyphenated, "count": count}
-        )
-
-        # Stop if we've reached max_words
-        if max_words and len(hyphenation_data) >= max_words:
-            break
-
-    print(f"Generated {len(hyphenation_data)} hyphenated words")
-
-    # Write output file
-    print(f"Writing to: {output_file}")
-    with open(output_file, "w", encoding="utf-8") as f:
-        # Write header with metadata
-        f.write(f"# Hyphenation Test Data\n")
-        f.write(f"# Source: {Path(input_file).name}\n")
-        f.write(f"# Language: {language}\n")
-        f.write(f"# Min prefix: {min_prefix}\n")
-        f.write(f"# Min suffix: {min_suffix}\n")
-        f.write(f"# Total words: {len(hyphenation_data)}\n")
-        f.write(f"# Format: word | hyphenated_form | frequency_in_source\n")
-        f.write(f"#\n")
-        f.write(f"# Hyphenation points are marked with '='\n")
-        f.write(f"# Example: Silbentrennung -> Sil=ben=tren=nung\n")
-        f.write(f"#\n\n")
-
-        # Write data
-        for item in hyphenation_data:
-            f.write(f"{item['word']}|{item['hyphenated']}|{item['count']}\n")
-
-    print("Done!")
-
-    # Print some statistics
-    print("\n=== Statistics ===")
-    print(f"Total unique words extracted: {len(word_counts)}")
-    print(f"Words with hyphenation points: {len(hyphenation_data)}")
-    print(
-        f"Average hyphenation points per word: {sum(h['hyphenated'].count('=') for h in hyphenation_data) / len(hyphenation_data):.2f}"
-    )
-
-    # Print some examples
-    print("\n=== Examples (first 10) ===")
-    for item in hyphenation_data[:10]:
-        print(
-            f"  {item['word']:20} -> {item['hyphenated']:30} (appears {item['count']}x)"
-        )
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Generate hyphenation test data from a text file",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-  # Generate test data from a German book
-  python generate_hyphenation_test_data.py ../data/books/bobiverse_1.txt hyphenation_test_data.txt
-  
-  # Limit to 500 most common words
-  python generate_hyphenation_test_data.py ../data/books/bobiverse_1.txt hyphenation_test_data.txt --max-words 500
-  
-  # Use English hyphenation (when available)
-  python generate_hyphenation_test_data.py book.txt test_en.txt --language en_US
-        """,
-    )
-
-    parser.add_argument("input_file", help="Input text file to extract words from")
-    parser.add_argument("output_file", help="Output file for hyphenation test data")
-    parser.add_argument(
-        "--language", default="de_DE", help="Language code (default: de_DE)"
-    )
-    parser.add_argument(
-        "--min-length", type=int, default=6, help="Minimum word length (default: 6)"
-    )
-    parser.add_argument(
-        "--max-words",
-        type=int,
-        default=5000,
-        help="Maximum number of words to include (default: 5000)",
-    )
-    parser.add_argument(
-        "--min-prefix",
-        type=int,
-        default=2,
-        help="Minimum characters permitted before the first hyphen (default: 2)",
-    )
-    parser.add_argument(
-        "--min-suffix",
-        type=int,
-        default=2,
-        help="Minimum characters permitted after the last hyphen (default: 2)",
-    )
-
-    args = parser.parse_args()
-
-    generate_hyphenation_data(
-        args.input_file,
-        args.output_file,
-        language=args.language,
-        min_length=args.min_length,
-        max_words=args.max_words,
-        min_prefix=args.min_prefix,
-        min_suffix=args.min_suffix,
-    )
-
-
-if __name__ == "__main__":
-    main()