feat: Arabic/Farsi/Urdu bidi reordering and contextual shaping — PR 1/3 (#2541)

Co-authored-by: Uri Tauber <uritaube@gmail.com>
2026-07-13 16:02:21 +03:00
parent 552b2683e6
commit 932a472835
18 changed files with 1087 additions and 62 deletions
@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
-project(crosspoint_reader_tests CXX)
+project(crosspoint_reader_tests C CXX)

 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -43,3 +43,5 @@ add_subdirectory(release_json_parser)
 add_subdirectory(differential_rounding)
 add_subdirectory(hyphenation_eval)
 add_subdirectory(utf8_compose)
+add_subdirectory(minibidi_arabic)
+add_subdirectory(combining_marks)
@@ -0,0 +1,14 @@
+add_executable(CombiningMarkAnchorTest
+  CombiningMarkAnchorTest.cpp
+)
+
+target_include_directories(CombiningMarkAnchorTest PRIVATE
+  ${REPO_ROOT}/lib/EpdFont
+)
+
+target_link_libraries(CombiningMarkAnchorTest PRIVATE
+  crosspoint_test_common
+  GTest::gtest_main
+)
+
+gtest_discover_tests(CombiningMarkAnchorTest)
@@ -0,0 +1,73 @@
+#include <gtest/gtest.h>
+
+#include "lib/EpdFont/EpdFontData.h"
+
+// ============================================================================
+// Anchor selection and placement for combining marks without GPOS tables.
+//
+// Hebrew niqqud whose identity depends on position must not use the default
+// centre-and-raise heuristic: dagesh sits inside the letter body, the
+// shin/sin dots sit over the letter's right/left arm, and holam hangs over
+// the left corner (see PR #2541 review feedback).
+// ============================================================================
+
+using combiningMark::Anchor;
+using combiningMark::anchorFor;
+using combiningMark::anchorOver;
+using combiningMark::anchorOverRotated90CW;
+using combiningMark::raiseAboveBase;
+
+TEST(AnchorFor, PositionSensitiveNiqqud) {
+  EXPECT_EQ(anchorFor(0x05BC), Anchor::CenterNative);  // dagesh/mapiq
+  EXPECT_EQ(anchorFor(0x05BA), Anchor::CenterNative);  // holam haser for vav
+  EXPECT_EQ(anchorFor(0x05C1), Anchor::RightNative);   // shin dot
+  EXPECT_EQ(anchorFor(0x05C2), Anchor::LeftNative);    // sin dot
+  EXPECT_EQ(anchorFor(0x05B9), Anchor::LeftNative);    // holam
+}
+
+TEST(AnchorFor, EverythingElseKeepsCentreRaisedDefault) {
+  EXPECT_EQ(anchorFor(0x05B0), Anchor::CenterRaised);  // Hebrew sheva
+  EXPECT_EQ(anchorFor(0x05B7), Anchor::CenterRaised);  // Hebrew patach
+  EXPECT_EQ(anchorFor(0x064E), Anchor::CenterRaised);  // Arabic fatha
+  EXPECT_EQ(anchorFor(0x0651), Anchor::CenterRaised);  // Arabic shadda
+  EXPECT_EQ(anchorFor(0x0301), Anchor::CenterRaised);  // combining acute
+}
+
+// Base glyph: cursor 100, left 1, width 12.  Mark: left 2, width 4.
+TEST(AnchorOver, HorizontalPlacementPerAnchor) {
+  // Centered: base bitmap spans [101, 113), centre 107; mark starts at 105.
+  EXPECT_EQ(anchorOver(Anchor::CenterRaised, 100, 1, 12, 2, 4), 105 - 2);
+  EXPECT_EQ(anchorOver(Anchor::CenterNative, 100, 1, 12, 2, 4), 105 - 2);
+  // Left-aligned: mark bitmap starts at the base bitmap's left edge (101).
+  EXPECT_EQ(anchorOver(Anchor::LeftNative, 100, 1, 12, 2, 4), 101 - 2);
+  // Right-aligned: mark bitmap ends at the base bitmap's right edge (113).
+  EXPECT_EQ(anchorOver(Anchor::RightNative, 100, 1, 12, 2, 4), 109 - 2);
+}
+
+// The rotated coordinate system inverts every left/width term, so the mark's
+// offset from the base cursor must be the exact mirror of the unrotated one.
+TEST(AnchorOverRotated90CW, MirrorsUnrotatedOffsets) {
+  for (const Anchor anchor : {Anchor::CenterRaised, Anchor::CenterNative, Anchor::LeftNative, Anchor::RightNative}) {
+    const int offset = anchorOver(anchor, 100, 1, 12, 2, 4) - 100;
+    EXPECT_EQ(anchorOverRotated90CW(anchor, 100, 1, 12, 2, 4), 100 - offset);
+  }
+}
+
+TEST(RaiseAboveBase, NativeAnchorsKeepFontDesignedHeight) {
+  // A dagesh-like dot designed inside the letter body (top 8 of a base whose
+  // top is 12) must NOT be hoisted above the letter.
+  EXPECT_EQ(raiseAboveBase(Anchor::CenterNative, 8, 3, 12), 0);
+  // Shin/sin dots overlapping the letter's top must not be pushed clear.
+  EXPECT_EQ(raiseAboveBase(Anchor::RightNative, 13, 3, 12), 0);
+  EXPECT_EQ(raiseAboveBase(Anchor::LeftNative, 13, 3, 12), 0);
+}
+
+TEST(RaiseAboveBase, CentreRaisedBehaviourUnchanged) {
+  // Above-baseline mark colliding with the base: raised to restore a 1px gap.
+  // gap = markTop - markHeight - baseTop = 10 - 3 - 12 = -5  ->  raise 6.
+  EXPECT_EQ(raiseAboveBase(Anchor::CenterRaised, 10, 3, 12), 6);
+  // Already clear of the base: no raise.
+  EXPECT_EQ(raiseAboveBase(Anchor::CenterRaised, 16, 3, 12), 0);
+  // Below-baseline mark (kasra, cedilla): stays at font-native position.
+  EXPECT_EQ(raiseAboveBase(Anchor::CenterRaised, 2, 4, 12), 0);
+}
@@ -0,0 +1,19 @@
+add_executable(MiniBidiArabicTest
+  MiniBidiArabicTest.cpp
+  ${REPO_ROOT}/lib/MiniBidi/minibidi.c
+  ${REPO_ROOT}/lib/MiniBidi/BidiUtils.cpp
+  ${REPO_ROOT}/lib/Utf8/Utf8.cpp
+)
+
+target_include_directories(MiniBidiArabicTest PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/stubs  # no-op Logging.h (real one needs Arduino)
+  ${REPO_ROOT}/lib/MiniBidi
+  ${REPO_ROOT}/lib/Utf8
+)
+
+target_link_libraries(MiniBidiArabicTest PRIVATE
+  crosspoint_test_common
+  GTest::gtest_main
+)
+
+gtest_discover_tests(MiniBidiArabicTest)
@@ -0,0 +1,254 @@
+// MiniBidiArabicTest — Arabic/Farsi/Urdu bidi + contextual shaping.
+//
+// Exercises the full BidiUtils::applyBidiVisual() pipeline (the single code
+// path shared by GfxRenderer::getTextWidth() and drawText()): UAX#9
+// reordering (do_bidi) followed by mintty-ported Arabic shaping (do_shape).
+//
+// Inputs are built from codepoint arrays in LOGICAL order; expectations are
+// codepoint arrays in VISUAL order (left to right, as drawn on screen).
+// Presentation-form expectations were derived by hand from Unicode
+// ArabicShaping.txt joining types and the Arabic Presentation Forms-A/B
+// blocks of UnicodeData.txt.
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "BidiUtils.h"
+#include "Utf8.h"
+
+namespace {
+
+std::string encode(const std::vector<uint32_t>& codepoints) {
+  std::string utf8;
+  for (const uint32_t cp : codepoints) utf8AppendCodepoint(cp, utf8);
+  return utf8;
+}
+
+std::vector<uint32_t> decode(const std::string& utf8) {
+  std::vector<uint32_t> codepoints;
+  auto* p = reinterpret_cast<const unsigned char*>(utf8.c_str());
+  while (*p) {
+    const uint32_t cp = utf8NextCodepoint(&p);
+    if (!cp) break;
+    codepoints.push_back(cp);
+  }
+  return codepoints;
+}
+
+// Runs the shared bidi+shape pipeline with auto-detected paragraph direction
+// and returns the visual-order codepoints.
+std::vector<uint32_t> shapeVisual(const std::vector<uint32_t>& logical) {
+  const std::string in = encode(logical);
+  std::string out;
+  if (!BidiUtils::applyBidiVisual(in.c_str(), out, /*paragraphLevel=*/-1)) {
+    return decode(in);  // pipeline declined: unchanged text
+  }
+  return decode(out);
+}
+
+using CP = std::vector<uint32_t>;
+
+}  // namespace
+
+/* ── Core Arabic contextual forms ────────────────────────────────────── */
+
+// A single letter renders in its isolated presentation form.
+TEST(ArabicShaping, SingleLetterIsolated) {
+  EXPECT_EQ(shapeVisual({0x0628}), (CP{0xFE8F}));  // ب
+}
+
+// دار: dal and alef are right-joining, so no letter connects forward —
+// all three stay isolated. Visual order is reversed.
+TEST(ArabicShaping, RightJoinersStayIsolated) {
+  EXPECT_EQ(shapeVisual({0x062F, 0x0627, 0x0631}), (CP{0xFEAD, 0xFE8D, 0xFEA9}));
+}
+
+// بيت: initial + medial + final chain of dual-joining letters.
+TEST(ArabicShaping, DualJoiningChain) {
+  EXPECT_EQ(shapeVisual({0x0628, 0x064A, 0x062A}), (CP{0xFE96, 0xFEF4, 0xFE91}));
+}
+
+// محمد: medial forms across four letters ending in right-joining dal.
+TEST(ArabicShaping, MedialForms) {
+  EXPECT_EQ(shapeVisual({0x0645, 0x062D, 0x0645, 0x062F}), (CP{0xFEAA, 0xFEE4, 0xFEA4, 0xFEE3}));
+}
+
+// Tatweel (U+0640) is join-causing and passes through unchanged.
+TEST(ArabicShaping, TatweelJoinsBothSides) {
+  EXPECT_EQ(shapeVisual({0x0628, 0x0640, 0x062A}), (CP{0xFE96, 0x0640, 0xFE91}));
+}
+
+/* ── Lam-Alef ligatures ──────────────────────────────────────────────── */
+
+// All four Lam-Alef variants, isolated (nothing joins into the Lam).
+// The absorbed Alef must be collapsed out of the output entirely.
+TEST(ArabicShaping, LamAlefIsolatedVariants) {
+  EXPECT_EQ(shapeVisual({0x0644, 0x0627}), (CP{0xFEFB}));  // لا
+  EXPECT_EQ(shapeVisual({0x0644, 0x0622}), (CP{0xFEF5}));  // لآ
+  EXPECT_EQ(shapeVisual({0x0644, 0x0623}), (CP{0xFEF7}));  // لأ
+  EXPECT_EQ(shapeVisual({0x0644, 0x0625}), (CP{0xFEF9}));  // لإ
+}
+
+// All four Lam-Alef variants in final form (a joining letter precedes Lam).
+TEST(ArabicShaping, LamAlefFinalVariants) {
+  EXPECT_EQ(shapeVisual({0x0628, 0x0644, 0x0627}), (CP{0xFEFC, 0xFE91}));
+  EXPECT_EQ(shapeVisual({0x0628, 0x0644, 0x0622}), (CP{0xFEF6, 0xFE91}));
+  EXPECT_EQ(shapeVisual({0x0628, 0x0644, 0x0623}), (CP{0xFEF8, 0xFE91}));
+  EXPECT_EQ(shapeVisual({0x0628, 0x0644, 0x0625}), (CP{0xFEFA, 0xFE91}));
+}
+
+// The specific bug flagged in PR #2398 review: a diacritic between Lam and
+// Alef must not break the ligature. The fatha stays in the stream (a
+// zero-advance overlay at render time); the Alef is absorbed. Per UAX#9 L3
+// the mark is emitted after the ligature it decorates.
+TEST(ArabicShaping, LamAlefWithDiacriticBetween) {
+  EXPECT_EQ(shapeVisual({0x0644, 0x064E, 0x0627}), (CP{0xFEFB, 0x064E}));
+}
+
+/* ── Harakat (diacritics) ────────────────────────────────────────────── */
+
+// كَتَبَ fully vocalized: harakat are transparent for joining — the letter
+// skeleton shapes exactly as كتب — and remain in the output stream. UAX#9
+// rule L3: each mark is emitted after its base, so the renderer can overlay
+// it on the most recently drawn glyph.
+TEST(ArabicShaping, VocalizedTextJoinsAcrossHarakat) {
+  EXPECT_EQ(shapeVisual({0x0643, 0x064E, 0x062A, 0x064E, 0x0628, 0x064E}),
+            (CP{0xFE90, 0x064E, 0xFE98, 0x064E, 0xFEDB, 0x064E}));
+}
+
+// Stacked marks (shadda + fatha on one base) keep their logical order after
+// the base, so above-base stacking renders bottom-up as authored.
+TEST(ArabicShaping, StackedMarksFollowBaseInLogicalOrder) {
+  EXPECT_EQ(shapeVisual({0x0628, 0x0651, 0x064E}), (CP{0xFE8F, 0x0651, 0x064E}));
+}
+
+// Hebrew niqqud rides the same L3 path: mark follows its base after the
+// RTL run is reversed.
+TEST(HebrewShaping, NiqqudFollowsBase) {
+  EXPECT_EQ(shapeVisual({0x05E9, 0x05B0, 0x05DC}), (CP{0x05DC, 0x05E9, 0x05B0}));
+}
+
+/* ── Farsi extra letters ─────────────────────────────────────────────── */
+
+// Peh joins like beh, using Presentation Forms-A codepoints.
+TEST(FarsiShaping, PehContextualForms) { EXPECT_EQ(shapeVisual({0x067E, 0x067E}), (CP{0xFB57, 0xFB58})); }
+
+// گاز: gaf initial, alef final, jeh-group zain isolated.
+TEST(FarsiShaping, GafWord) { EXPECT_EQ(shapeVisual({0x06AF, 0x0627, 0x0632}), (CP{0xFEAF, 0xFE8E, 0xFB94})); }
+
+// سیب: Farsi yeh (U+06CC) takes its medial Presentation Forms-A form.
+TEST(FarsiShaping, FarsiYehMedial) { EXPECT_EQ(shapeVisual({0x0633, 0x06CC, 0x0628}), (CP{0xFE90, 0xFBFF, 0xFEB3})); }
+
+/* ── Urdu extra letters ──────────────────────────────────────────────── */
+
+// ٹ (U+0679) and ڑ (U+0691) were specifically flagged in PR #2398 review as
+// missing from a literal mintty port; ڈ (U+0688) completes the retroflex set.
+TEST(UrduShaping, RetroflexLetters) { EXPECT_EQ(shapeVisual({0x0679, 0x0688, 0x0691}), (CP{0xFB8C, 0xFB89, 0xFB68})); }
+
+// میں: noon ghunna (U+06BA) is dual-joining but only has isolated/final
+// presentation forms — final position works.
+TEST(UrduShaping, NoonGhunnaFinal) { EXPECT_EQ(shapeVisual({0x0645, 0x06CC, 0x06BA}), (CP{0xFB9F, 0xFBFF, 0xFEE3})); }
+
+// ہے: heh goal initial + yeh barree final.
+TEST(UrduShaping, HehGoalYehBarree) { EXPECT_EQ(shapeVisual({0x06C1, 0x06D2}), (CP{0xFBAF, 0xFBA8})); }
+
+/* ── Sindhi / Pashto / Kurdish samples ───────────────────────────────── */
+
+// Sindhi ٻار: beeh (U+067B) initial form from Presentation Forms-A.
+TEST(SindhiShaping, BeehInitial) { EXPECT_EQ(shapeVisual({0x067B, 0x0627, 0x0631}), (CP{0xFEAD, 0xFE8E, 0xFB54})); }
+
+// Pashto ښه: seen-with-dots (U+069A) has a joining type but NO presentation
+// forms — it keeps its base codepoint while its neighbour still takes the
+// correct joined form.
+TEST(PashtoShaping, LetterWithoutPresentationFormsFallsBack) {
+  EXPECT_EQ(shapeVisual({0x069A, 0x0647}), (CP{0xFEEA, 0x069A}));
+}
+
+// Pashto کور: keheh initial, waw final, reh isolated.
+TEST(PashtoShaping, KehehWord) { EXPECT_EQ(shapeVisual({0x06A9, 0x0648, 0x0631}), (CP{0xFEAD, 0xFEEE, 0xFB90})); }
+
+// Kurdish ڕۆژ: all right-joining; ڕ (U+0695) has no presentation forms and
+// stays as its base codepoint, ۆ and ژ take Presentation Forms-A isolated.
+TEST(KurdishShaping, RightJoiningLetters) {
+  EXPECT_EQ(shapeVisual({0x0695, 0x06C6, 0x0698}), (CP{0xFB8A, 0xFBD9, 0x0695}));
+}
+
+/* ── ZWJ / ZWNJ joining formatters ───────────────────────────────────── */
+
+// ZWNJ blocks joining across it but leaves the outer sides to normal
+// contextual rules (Farsi morphology, e.g. می‌خواهم). The formatter itself
+// must be filtered from the output. Here meem joins into yeh (initial +
+// final) while the ZWNJ keeps yeh from connecting to khah.
+TEST(JoinerShaping, ZwnjBreaksJoin) {
+  // می‌خ: trailing khah has no forward partner → isolated.
+  EXPECT_EQ(shapeVisual({0x0645, 0x06CC, 0x200C, 0x062E}), (CP{0xFEA5, 0xFBFD, 0xFEE3}));
+  // می‌خو: waw follows khah → khah initial, waw final.
+  EXPECT_EQ(shapeVisual({0x0645, 0x06CC, 0x200C, 0x062E, 0x0648}), (CP{0xFEEE, 0xFEA7, 0xFBFD, 0xFEE3}));
+}
+
+// ZWJ forces a join where none would occur: a lone beh followed by ZWJ takes
+// initial form; preceded by ZWJ it takes final form.
+TEST(JoinerShaping, ZwjForcesJoin) {
+  EXPECT_EQ(shapeVisual({0x0628, 0x200D}), (CP{0xFE91}));
+  EXPECT_EQ(shapeVisual({0x200D, 0x0628}), (CP{0xFE90}));
+}
+
+/* ── Mixed-direction text ────────────────────────────────────────────── */
+
+// كتاب abc 123 — RTL paragraph: the Arabic (shaped) ends up rightmost;
+// "abc 123" resolves to a single LTR run (rule W7 turns the European
+// numerals L after the strong L of "abc") and is placed as one block to
+// its left, keeping internal left-to-right order.
+TEST(MixedDirection, ArabicLatinNumerals) {
+  EXPECT_EQ(shapeVisual({0x0643, 0x062A, 0x0627, 0x0628, ' ', 'a', 'b', 'c', ' ', '1', '2', '3'}),
+            (CP{'a', 'b', 'c', ' ', '1', '2', '3', ' ', 0xFE8F, 0xFE8E, 0xFE98, 0xFEDB}));
+}
+
+// كتاب ١٢٣ — Arabic-Indic digits keep logical order (leftmost run reads
+// ١٢٣, not reversed).
+TEST(MixedDirection, ArabicIndicNumerals) {
+  EXPECT_EQ(shapeVisual({0x0643, 0x062A, 0x0627, 0x0628, ' ', 0x0661, 0x0662, 0x0663}),
+            (CP{0x0661, 0x0662, 0x0663, ' ', 0xFE8F, 0xFE8E, 0xFE98, 0xFEDB}));
+}
+
+// Watch-item regression probe for the "random spaces" report from #2398
+// testing: spaces must be neither duplicated nor dropped by the pipeline.
+TEST(MixedDirection, SpaceCountPreserved) {
+  const CP visual = shapeVisual({0x0643, 0x062A, 0x0627, 0x0628, ' ', 0x0642, 0x0644, 0x0645, ' ', 'o', 'k'});
+  int spaces = 0;
+  for (const uint32_t cp : visual) spaces += (cp == ' ');
+  EXPECT_EQ(spaces, 2);
+  EXPECT_EQ(visual.size(), 11u);  // 9 letters + 2 spaces, nothing absorbed or invented
+}
+
+/* ── Regression: existing behaviour unchanged ────────────────────────── */
+
+// Hebrew reorders but never shapes.
+TEST(Regression, HebrewUntouchedByShaper) {
+  EXPECT_EQ(shapeVisual({0x05E9, 0x05DC, 0x05D5, 0x05DD}), (CP{0x05DD, 0x05D5, 0x05DC, 0x05E9}));
+}
+
+// Pure Latin text passes through unchanged.
+TEST(Regression, LatinPassthrough) { EXPECT_EQ(shapeVisual({'h', 'e', 'l', 'l', 'o'}), (CP{'h', 'e', 'l', 'l', 'o'})); }
+
+/* ── isTransparentMark ───────────────────────────────────────────────── */
+
+TEST(TransparentMark, RtlMarksAreTransparent) {
+  EXPECT_TRUE(BidiUtils::isTransparentMark(0x05B0));  // Hebrew sheva
+  EXPECT_TRUE(BidiUtils::isTransparentMark(0x0591));  // Hebrew accent etnahta
+  EXPECT_TRUE(BidiUtils::isTransparentMark(0x064E));  // Arabic fatha
+  EXPECT_TRUE(BidiUtils::isTransparentMark(0x0651));  // Arabic shadda
+  EXPECT_TRUE(BidiUtils::isTransparentMark(0x0670));  // superscript alef
+  EXPECT_TRUE(BidiUtils::isTransparentMark(0x06D6));  // Quranic annotation
+}
+
+TEST(TransparentMark, LettersAndLatinMarksAreNot) {
+  EXPECT_FALSE(BidiUtils::isTransparentMark(0x0300));  // Latin combining grave — different path
+  EXPECT_FALSE(BidiUtils::isTransparentMark(0x0628));  // Arabic beh
+  EXPECT_FALSE(BidiUtils::isTransparentMark(0x05D0));  // Hebrew alef
+  EXPECT_FALSE(BidiUtils::isTransparentMark(0x0661));  // Arabic-Indic digit
+  EXPECT_FALSE(BidiUtils::isTransparentMark('a'));
+}
@@ -0,0 +1,8 @@
+#pragma once
+
+// Host-test stub for lib/Logging/Logging.h, which depends on Arduino's
+// HardwareSerial and cannot compile on the host. Logging is a no-op here.
+
+#define LOG_ERR(origin, format, ...)
+#define LOG_INF(origin, format, ...)
+#define LOG_DBG(origin, format, ...)