fix(core): address white-box review feedback

- Fix unicode surrogate pairs and emoji slicing via Array.from in ORP calculator - Add slot smoothing constraint to pivot element to prevent horizontal jitter - Implement adaptive timestamp compensation and visibilitychange auto-pause - Slim storage layer by updating only STORE_META on progress changes with memory fallback - Align EPUB chapter indices to token array indices instead of raw character offsets - Support standalone emoji tokens via Extended_Pictographic regex - Code-split JSZip via dynamic import to optimize initial bundle size Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-20 08:37:32 +08:00
parent c6f541ed94
commit 1f7a6a2054
7 changed files with 397 additions and 132 deletions
--- a/src/core/epub.ts
+++ b/src/core/epub.ts
@@ -1,14 +1,23 @@
-import JSZip from 'jszip';
-import type { Chapter } from '../types';
+export interface EpubChapterItem {
+  id: string;
+  title: string;
+  charOffset: number;
+}

 export interface ParsedEpub {
  title: string;
  author: string;
  fullText: string;
-  chapters: Chapter[];
+  rawChapters: EpubChapterItem[];
 }

+/**
+ * Client-side EPUB parser with dynamic JSZip import to reduce initial bundle size.
+ */
 export async function parseEpubFile(file: File | Blob): Promise<ParsedEpub> {
+  // Dynamic import JSZip to split ~100KB off the initial load bundle
+  const { default: JSZip } = await import('jszip');
+
  const arrayBuffer = await file.arrayBuffer();
  const zip = await JSZip.loadAsync(arrayBuffer);

@@ -39,8 +48,11 @@ export async function parseEpubFile(file: File | Blob): Promise<ParsedEpub> {
  const opfDoc = domParser.parseFromString(opfXml, 'application/xml');

  // Metadata
-  const title = opfDoc.querySelector('title, dc\\:title')?.textContent?.trim() || (file instanceof File ? file.name.replace(/\.epub$/i, '') : '未命名书籍');
-  const author = opfDoc.querySelector('creator, dc\\:creator')?.textContent?.trim() || '未知作者';
+  const title =
+    opfDoc.querySelector('title, dc\\:title')?.textContent?.trim() ||
+    (file instanceof File ? file.name.replace(/\.epub$/i, '') : '未命名书籍');
+  const author =
+    opfDoc.querySelector('creator, dc\\:creator')?.textContent?.trim() || '未知作者';

  // Manifest: id -> href
  const manifestMap = new Map<string, string>();
@@ -48,7 +60,6 @@ export async function parseEpubFile(file: File | Blob): Promise<ParsedEpub> {
    const id = item.getAttribute('id');
    const href = item.getAttribute('href');
    if (id && href) {
-      // Decode URL encoding
      manifestMap.set(id, decodeURIComponent(href));
    }
  });
@@ -62,7 +73,7 @@ export async function parseEpubFile(file: File | Blob): Promise<ParsedEpub> {
    }
  });

-  const chapters: Chapter[] = [];
+  const rawChapters: EpubChapterItem[] = [];
  let fullText = '';
  let chapterIndex = 1;

@@ -70,7 +81,6 @@ export async function parseEpubFile(file: File | Blob): Promise<ParsedEpub> {
    const relativeHref = manifestMap.get(idref);
    if (!relativeHref) continue;

-    // Resolve full path inside zip
    const filePath = opfDir ? resolvePath(opfDir, relativeHref) : relativeHref;
    const chapterFile = zip.file(filePath);
    if (!chapterFile) continue;
@@ -87,7 +97,6 @@ export async function parseEpubFile(file: File | Blob): Promise<ParsedEpub> {

    // Extract text content cleanly
    const body = chapterDoc.body || chapterDoc.documentElement;
-    // Remove scripts and styles
    body.querySelectorAll('script, style, svg').forEach((el) => el.remove());

    const paragraphs: string[] = [];
@@ -105,10 +114,11 @@ export async function parseEpubFile(file: File | Blob): Promise<ParsedEpub> {

    const chapterText = paragraphs.join('\n\n');
    if (chapterText.trim().length > 0) {
-      chapters.push({
+      const startCharOffset = fullText.length;
+      rawChapters.push({
        id: `chap-${chapterIndex}`,
        title: chapterTitle,
-        startIndex: fullText.length,
+        charOffset: startCharOffset,
      });

      fullText += (fullText ? '\n\n' : '') + chapterText;
@@ -124,7 +134,7 @@ export async function parseEpubFile(file: File | Blob): Promise<ParsedEpub> {
    title,
    author,
    fullText,
-    chapters,
+    rawChapters,
  };
 }

--- a/src/core/orp.ts
+++ b/src/core/orp.ts
@@ -1,6 +1,8 @@
 /**
 * Optimal Recognition Point (ORP) Calculation for Spritz RSVP Reading.
 * Fixates the eye at ~30%-35% into the word where human ocular fixation naturally rests.
+ * Uses unicode-safe code point segmentation (Array.from) to eliminate surrogate pair
+ * and emoji splitting defects.
 */

 const CJK_REGEX = /[一-龥぀-ヿ가-힯]/;
@@ -15,9 +17,11 @@ export function isCJKWord(word: string): boolean {

 /**
 * Calculates the 0-based index of the ORP character in cleanWord.
+ * Based on unicode code-point length (Array.from(cleanWord).length).
 */
 export function calculateOrpIndex(cleanWord: string): number {
-  const len = cleanWord.length;
+  const chars = Array.from(cleanWord);
+  const len = chars.length;
  if (len <= 1) return 0;

  if (isCJKWord(cleanWord)) {
@@ -45,6 +49,7 @@ export interface SplitWord {
 /**
 * Splits a token into prefix, pivot (ORP character), and suffix.
 * Handles leading punctuation (e.g. "“", "《", "(") and trailing punctuation.
+ * Operates purely on unicode code points to prevent corrupting surrogate pairs.
 */
 export function splitOrp(
  coreWord: string,
@@ -60,10 +65,11 @@ export function splitOrp(
    };
  }

+  const chars = Array.from(coreWord);
  const orpIndex = calculateOrpIndex(coreWord);
-  const prefix = leadingPunct + coreWord.slice(0, orpIndex);
-  const pivot = coreWord[orpIndex] || '';
-  const suffix = coreWord.slice(orpIndex + 1) + trailingPunct;
+  const prefix = leadingPunct + chars.slice(0, orpIndex).join('');
+  const pivot = chars[orpIndex] || '';
+  const suffix = chars.slice(orpIndex + 1).join('') + trailingPunct;

  return {
    prefix,
--- a/src/core/tokenizer.ts
+++ b/src/core/tokenizer.ts
@@ -10,7 +10,10 @@ const SENTENCE_PUNCT_REGEX = /[.!?。！？…\n]/;
 // Clause/pause punctuation
 const CLAUSE_PUNCT_REGEX = /[,;:，；：、—–]/;

-// Chapter detection regex patterns
+// Emoji / Extended pictographic Unicode matcher
+const EMOJI_REGEX = /^\p{Extended_Pictographic}+$/u;
+
+// Chapter detection regex patterns for raw text / markdown
 const CHAPTER_REGEXES = [
  /^(#{1,3}\s+.+)$/m,                                      // # Markdown header
  /^(第[0-9一二三四五六七八九十百千万]+[章回卷节篇部集].*)$/m, // 第X章 ...
@@ -20,6 +23,13 @@ const CHAPTER_REGEXES = [
  /^(Act\s+[0-9IVXLCDM]+.*)$/im,                          // Act X ...
 ];

+export interface ChapterInputHint {
+  id?: string;
+  title: string;
+  charOffset?: number;
+  startIndex?: number;
+}
+
 export interface TokenizeResult {
  tokens: Token[];
  chapters: Chapter[];
@@ -27,14 +37,28 @@ export interface TokenizeResult {

 export function tokenizeText(
  rawText: string,
-  existingChapters?: Chapter[]
+  existingChapters?: ChapterInputHint[]
 ): TokenizeResult {
  if (!rawText || !rawText.trim()) {
    return { tokens: [], chapters: [] };
  }

-  const chapters: Chapter[] = existingChapters ? [...existingChapters] : [];
-  const autoDetectChapters = !existingChapters || existingChapters.length === 0;
+  // Determine chapter source
+  const hasExternalHints = existingChapters && existingChapters.length > 0;
+  const chapterHints: { id: string; title: string; charOffset: number }[] = [];
+
+  if (hasExternalHints) {
+    existingChapters.forEach((c, idx) => {
+      chapterHints.push({
+        id: c.id || `chap-${idx + 1}`,
+        title: c.title,
+        charOffset: c.charOffset ?? c.startIndex ?? 0,
+      });
+    });
+    chapterHints.sort((a, b) => a.charOffset - b.charOffset);
+  }
+
+  const autoDetectChapters = !hasExternalHints;

  let segmenter: Intl.Segmenter | null = null;
  try {
@@ -70,6 +94,7 @@ export function tokenizeText(
    }
  }

+  const chapters: Chapter[] = [];
  const tokens: Token[] = [];
  let pendingLeading = '';
  let newlineCount = 0;
@@ -82,8 +107,26 @@ export function tokenizeText(
    for (const seg of segments) {
      const { segment, isWordLike, index } = seg;

-      // Check if we reached a detected chapter boundary
-      if (
+      // Align chapters based on character offset -> exact Token index
+      if (hasExternalHints) {
+        while (
+          nextChapterIdx < chapterHints.length &&
+          index >= chapterHints[nextChapterIdx].charOffset
+        ) {
+          const hint = chapterHints[nextChapterIdx];
+          currentChapterIndex = chapters.length;
+          chapters.push({
+            id: hint.id,
+            title: hint.title,
+            startIndex: tokens.length, // True token index
+          });
+          nextChapterIdx++;
+          if (tokens.length > 0) {
+            tokens[tokens.length - 1].isParagraphEnd = true;
+            tokens[tokens.length - 1].isSentenceEnd = true;
+          }
+        }
+      } else if (
        autoDetectChapters &&
        nextChapterIdx < detectedChapterPositions.length &&
        index >= detectedChapterPositions[nextChapterIdx].index
@@ -93,7 +136,7 @@ export function tokenizeText(
        chapters.push({
          id: `chap-${chapters.length + 1}`,
          title: chapInfo.line,
-          startIndex: tokens.length,
+          startIndex: tokens.length, // True token index
        });
        nextChapterIdx++;
        if (tokens.length > 0) {
@@ -116,22 +159,49 @@ export function tokenizeText(
          continue;
        }

+        // Emoji compatibility: Extended_Pictographic symbols stand as full tokens
+        if (EMOJI_REGEX.test(trimmed)) {
+          const { prefix, pivot, suffix, orpIndex } = splitOrp(
+            trimmed,
+            pendingLeading,
+            ''
+          );
+          tokens.push({
+            id: tokens.length,
+            text: prefix + pivot + suffix,
+            cleanWord: trimmed,
+            prefix,
+            pivot,
+            suffix,
+            orpIndex,
+            isParagraphEnd: newlineCount >= 2,
+            isSentenceEnd: false,
+            isClauseEnd: false,
+            isLongWord: false,
+            chapterIndex: currentChapterIndex,
+          });
+          pendingLeading = '';
+          newlineCount = 0;
+          continue;
+        }
+
        if (LEADING_PUNCT_REGEX.test(trimmed)) {
          // Attaches to the NEXT token's prefix
          pendingLeading += trimmed;
-        } else {
+        } else if (tokens.length > 0) {
          // Trailing punctuation attached to the PREVIOUS token
-          if (tokens.length > 0) {
-            const prev = tokens[tokens.length - 1];
-            prev.text += trimmed;
-            prev.suffix += trimmed;
-            if (SENTENCE_PUNCT_REGEX.test(trimmed)) {
-              prev.isSentenceEnd = true;
-            }
-            if (CLAUSE_PUNCT_REGEX.test(trimmed)) {
-              prev.isClauseEnd = true;
-            }
+          const prev = tokens[tokens.length - 1];
+          prev.text += trimmed;
+          prev.suffix += trimmed;
+          if (SENTENCE_PUNCT_REGEX.test(trimmed)) {
+            prev.isSentenceEnd = true;
          }
+          if (CLAUSE_PUNCT_REGEX.test(trimmed)) {
+            prev.isClauseEnd = true;
+          }
+        } else {
+          // Leading punctuation or symbols at the start of text (don't discard)
+          pendingLeading += trimmed;
        }

        if (newlineCount >= 2 && tokens.length > 0) {
@@ -145,8 +215,8 @@ export function tokenizeText(
      if (!coreWord) continue;

      const isLong = isCJKWord(coreWord)
-        ? coreWord.length >= 4
-        : coreWord.length > 8;
+        ? Array.from(coreWord).length >= 4
+        : Array.from(coreWord).length > 8;

      const { prefix, pivot, suffix, orpIndex } = splitOrp(
        coreWord,
@@ -175,13 +245,13 @@ export function tokenizeText(
      newlineCount = 0;
    }
  } else {
-    // Regex fallback for environments without Intl.Segmenter
-    const wordPattern = /[一-龥]|[a-zA-Z0-9_À-ɏ]+|[^\s\w]/g;
+    // Fallback regex loop with Emoji support
+    const wordPattern = /\p{Extended_Pictographic}+|[一-龥]|[a-zA-Z0-9_À-ɏ]+|[^\s\w]/gu;
    let match: RegExpExecArray | null;

    while ((match = wordPattern.exec(rawText)) !== null) {
      const str = match[0];
-      if (/[一-龥a-zA-Z0-9]/.test(str)) {
+      if (EMOJI_REGEX.test(str) || /[一-龥a-zA-Z0-9]/.test(str)) {
        const { prefix, pivot, suffix, orpIndex } = splitOrp(
          str,
          pendingLeading,
@@ -198,26 +268,46 @@ export function tokenizeText(
          isParagraphEnd: newlineCount >= 2,
          isSentenceEnd: false,
          isClauseEnd: false,
-          isLongWord: str.length > 8,
+          isLongWord: Array.from(str).length > 8,
          chapterIndex: currentChapterIndex,
        });
        pendingLeading = '';
        newlineCount = 0;
      } else if (LEADING_PUNCT_REGEX.test(str)) {
        pendingLeading += str;
+      } else if (tokens.length > 0) {
+        const prev = tokens[tokens.length - 1];
+        prev.text += str;
+        prev.suffix += str;
+        if (SENTENCE_PUNCT_REGEX.test(str)) prev.isSentenceEnd = true;
+        if (CLAUSE_PUNCT_REGEX.test(str)) prev.isClauseEnd = true;
      } else {
-        if (tokens.length > 0) {
-          const prev = tokens[tokens.length - 1];
-          prev.text += str;
-          prev.suffix += str;
-          if (SENTENCE_PUNCT_REGEX.test(str)) prev.isSentenceEnd = true;
-          if (CLAUSE_PUNCT_REGEX.test(str)) prev.isClauseEnd = true;
-        }
+        pendingLeading += str;
      }
    }
  }

-  // If no chapters detected at all, supply default single chapter
+  // If text only consisted of pendingLeading symbols and no words
+  if (tokens.length === 0 && pendingLeading.trim().length > 0) {
+    const trimmed = pendingLeading.trim();
+    const { prefix, pivot, suffix, orpIndex } = splitOrp(trimmed, '', '');
+    tokens.push({
+      id: 0,
+      text: trimmed,
+      cleanWord: trimmed,
+      prefix,
+      pivot,
+      suffix,
+      orpIndex,
+      isParagraphEnd: false,
+      isSentenceEnd: false,
+      isClauseEnd: false,
+      isLongWord: false,
+      chapterIndex: 0,
+    });
+  }
+
+  // If no chapters detected, supply default single chapter
  if (chapters.length === 0 && tokens.length > 0) {
    chapters.push({
      id: 'chap-1',