Optimizer: split oversized paragraphs/chapters, strip fonts, extract data-URI images

crosspoint_reader/README.md

@@ -31,6 +31,17 @@ enabled (Preferences > Plugins > device config >
   width/height are stripped, SVG covers/wrapped images are unwrapped, OPF
   media-types and cover meta are fixed, the NCX identifier is synced, a small
   defensive stylesheet is injected, and the archive is re-zipped mimetype-first.
+- Text is restructured for the firmware's layout memory limits (optional,
+  "Split large chapters/paragraphs" checkbox, on by default): paragraphs larger
+  than ~1.6 KB are split into ~1.2 KB `<p>` siblings at sentence boundaries
+  (the firmware lays out a whole paragraph at once, holding every word in RAM,
+  so a single multi-KB paragraph can OOM the device even in a small file);
+  spine files larger than ~9.5 KB are split into ~7 KB files with the OPF
+  manifest/spine expanded and `href="...#fragment"` links remapped onto the
+  chunk holding the anchor; embedded fonts and `@font-face` rules are removed;
+  page-list navs are dropped; and base64 `data:` URI images are extracted into
+  real (optimized) image files. Every text transformation verifies that the
+  visible text is byte-identical and reverts itself on any mismatch or error.
 
 The target screen size comes from the device profile — **X4 = 480×800**,
 **X3 = 528×792** — which is auto-detected from the device's `/api/status`
@@ -56,6 +67,9 @@ never blocked.
    - **JPEG quality** — 1–100 (default 85). Lower = smaller files.
    - **Convert images to grayscale** — on by default (recommended for e-ink).
    - **Auto-crop uniform margins** — off by default; trims solid page borders.
+   - **Split large chapters/paragraphs, remove fonts (prevents out-of-memory)**
+     — on by default; restructures text for the firmware's layout memory limits
+     (see above). Disable it to keep the EPUB's file/paragraph structure as-is.
 4. Click **OK**, then **restart Calibre** if it was already running so the new
    settings take effect.
 5. Send a book to the device as usual (right-click → *Send to device*, or the

crosspoint_reader/config.py

@@ -32,6 +32,7 @@
 PREFS.defaults['optimize_grayscale'] = True
 PREFS.defaults['optimize_auto_crop'] = False
 PREFS.defaults['optimize_quality'] = 85
+PREFS.defaults['optimize_split'] = True
 PREFS.defaults['device_target'] = 'auto'  # 'auto' | 'X4' | 'X3'
 
 
@@ -53,6 +54,8 @@ def __init__(self):
         self.optimize = QCheckBox('Optimize EPUBs before transfer', self)
         self.optimize_grayscale = QCheckBox('Convert images to grayscale', self)
         self.optimize_auto_crop = QCheckBox('Auto-crop uniform margins', self)
+        self.optimize_split = QCheckBox(
+            'Split large chapters/paragraphs, remove fonts (prevents out-of-memory)', self)
         self.optimize_quality = QSpinBox(self)
         self.optimize_quality.setRange(1, 100)
         self.optimize_quality.setSuffix('%')
@@ -71,6 +74,7 @@ def __init__(self):
         self.optimize.setChecked(PREFS['optimize'])
         self.optimize_grayscale.setChecked(PREFS['optimize_grayscale'])
         self.optimize_auto_crop.setChecked(PREFS['optimize_auto_crop'])
+        self.optimize_split.setChecked(PREFS['optimize_split'])
         self.optimize_quality.setValue(PREFS['optimize_quality'])
         idx = self.device_target.findData(PREFS['device_target'])
         self.device_target.setCurrentIndex(idx if idx >= 0 else 0)
@@ -107,6 +111,7 @@ def __init__(self):
         layout.addRow('JPEG quality', self.optimize_quality)
         layout.addRow('', self.optimize_grayscale)
         layout.addRow('', self.optimize_auto_crop)
+        layout.addRow('', self.optimize_split)
 
         self.optimize.toggled.connect(self._sync_optimizer_enabled)
         self._sync_optimizer_enabled(self.optimize.isChecked())
@@ -135,12 +140,13 @@ def save(self):
         PREFS['optimize'] = bool(self.optimize.isChecked())
         PREFS['optimize_grayscale'] = bool(self.optimize_grayscale.isChecked())
         PREFS['optimize_auto_crop'] = bool(self.optimize_auto_crop.isChecked())
+        PREFS['optimize_split'] = bool(self.optimize_split.isChecked())
         PREFS['optimize_quality'] = int(self.optimize_quality.value())
         PREFS['device_target'] = self.device_target.currentData()
 
     def _sync_optimizer_enabled(self, enabled):
         for w in (self.optimize_grayscale, self.optimize_auto_crop,
-                  self.optimize_quality, self.device_target):
+                  self.optimize_split, self.optimize_quality, self.device_target):
             w.setEnabled(enabled)
 
     def _refresh_logs(self):

crosspoint_reader/driver.py

@@ -465,6 +465,7 @@ def _optimize_book(self, filepath, profile, step_cb=None):
             quality=PREFS['optimize_quality'],
             grayscale=PREFS['optimize_grayscale'],
             auto_crop=PREFS['optimize_auto_crop'],
+            split_text=PREFS['optimize_split'],
         )
 
         def _step(tag, message):

crosspoint_reader/optimizer.py

@@ -79,10 +79,14 @@ def resolve_profile(device_target, detected_device):
 
 
 class Options(object):
-    def __init__(self, quality=DEFAULT_JPEG_QUALITY, grayscale=True, auto_crop=False):
+    def __init__(self, quality=DEFAULT_JPEG_QUALITY, grayscale=True, auto_crop=False,
+                 split_text=True):
         self.quality = int(quality)
         self.grayscale = bool(grayscale)
         self.auto_crop = bool(auto_crop)
+        # Split oversized paragraphs/chapters and strip fonts for the
+        # low-RAM firmware layout engine (see textsplit.py).
+        self.split_text = bool(split_text)
 
 
 # ---------------------------------------------------------------------------
@@ -596,6 +600,14 @@ def log(tag, message):
     finally:
         zin.close()
 
+    if getattr(opts, 'split_text', False):
+        from .textsplit import split_epub_text
+        split_summary = split_epub_text(out_path, log, profile, opts)
+        summary['fixes'] += (split_summary.get('paras', 0)
+                             + split_summary.get('file_splits', 0)
+                             + split_summary.get('fonts', 0)
+                             + split_summary.get('dataimgs', 0))
+
     summary['new_size'] = os.path.getsize(out_path)
     summary['elapsed'] = time.time() - start
     saved = orig_size - summary['new_size']