added SuryaOCR batch mode; some fixes

README.md

@@ -59,7 +59,7 @@ src/image_ocr.py           ← Stage 2: OCR + Markdown
 | `--slice-auto` / `-a` | Автоопределение сетки | — |
 | `--output-dir` / `-o` | Каталог для страниц | `.` |
 | `--pre-rotate` / `-r` | Поворот: 90, 180, 270 | — |
-| `--border` / `-b` | Белая рамка (px) | 5 |
+| `--border` / `-b` | Белая рамка (px) | 10 |
 | `--unwarp` / `-u` | Выпрямить перспективные искажения (Canny → 4-угольный контур → warp) | OFF |
 | `--denoise` | Убрать шум сканера (Non-Local Means) | OFF |
 | `--post-crop` / `--no-post-crop` | Обрезка по контенту | ON |
@@ -105,8 +105,8 @@ python -m src.image_prepare -i old_scan.jpg --denoise -s 2:2 -b 10
 │                                                         (белая рамка /       │
 │                                                         отступ при обрезке   │
 │                                                         содержимого, по      │
-│                                                         умолчанию 5)         │
-│                                                         [default: 5]         │
+│                                                         умолчанию 10)        │
+│                                                         [default: 10]        │
 │ --unwarp      -u                                        Выпрямить            │
 │                                                         перспективные        │
 │                                                         искажения перед      │

src/image_ocr.py

@@ -53,32 +53,64 @@ IMG_EXTENSIONS: set[str] = {".jpg", ".jpeg", ".png", ".tiff", ".tif"}
 
 def _get_ocr_engine(name: str, main_lang: str):
     if name == "surya":
-        from src.ocr.surya_engine import surya_ocr_image
-        return surya_ocr_image, "Surya 2 VLM"
+        from src.ocr.surya_engine import surya_ocr_batch
+        return surya_ocr_batch, "Surya 2 VLM", True
     if name == "external-qwen3":
         from src.ocr.external_vlm_engine import external_vlm_ocr_image
-        return external_vlm_ocr_image, "Qwen3.8 Max (external VLM)"
-    return functools.partial(ocr_image, main_lang=main_lang), "PP-StructureV3"
+        return external_vlm_ocr_image, "Qwen3.8 Max (external VLM)", False
+    return functools.partial(ocr_image, main_lang=main_lang), "PP-StructureV3", False
 
 
-def _merge_markdown_files(output_dir: Path, md_files: list[Path]) -> None:
+def _merge_markdown_files(output_dir: Path, input_name: str, md_files: list[Path]) -> None:
     merged_parts: list[str] = []
     for md_path in sorted(md_files):
         content = md_path.read_text(encoding="utf-8")
         merged_parts.append(f"<!-- page: {md_path.stem} -->\n\n{content}")
     if merged_parts:
         merged = "# Merged document\n\n" + "\n\n---\n\n".join(merged_parts)
-        merged_path = output_dir / (output_dir.name + MD_EXTENSION)
+        merged_path = output_dir / (input_name + MD_EXTENSION)
         merged_path.write_text(merged, encoding="utf-8")
         logger.info("Объединённый документ: %s", merged_path)
 
 
+def _log_low_confidence(raw_json: dict) -> None:
+    layout = raw_json.get("layout_det_res", {}).get("boxes", [])
+    if not layout:
+        return
+    low = [(b.get("label", "?"), round(b.get("score", 0), 3)) for b in layout if b.get("score", 1) < 0.5]
+    if low:
+        labels = ", ".join(f"{lbl}({sc})" for lbl, sc in low)
+        logger.warning("Низкая уверенность: %d блоков — %s", len(low), labels)
+
+
+def _apply_vlm(blocks, input_path: Path) -> None:
+    from src.postprocess.corrector import apply_corrector
+    from src.postprocess.vlm_corrector import vlm_correct_block
+    corrector = functools.partial(vlm_correct_block, image_path=str(input_path.resolve()))
+    apply_corrector(blocks, corrector, "VLM")
+
+
+def _save_result(page, input_path: Path, output_dir: Path) -> None:
+    json_path = output_dir / (input_path.stem + ".json")
+    if not json_path.exists():
+        json_payload = page.raw_json or {
+            "blocks": [{"label": b.label, "content": b.content, "bbox": list(b.bbox)}
+                       for b in page.blocks]
+        }
+        json_path.write_text(json.dumps(json_payload, ensure_ascii=False, indent=2), encoding="utf-8")
+        logger.debug("JSON сохранён: %s", json_path)
+
+    md_path = output_dir / (input_path.stem + MD_EXTENSION)
+    generate_markdown(page.blocks, md_path)
+    logger.debug("Markdown сохранён: %s", md_path)
+    logger.info("%s → %s, %s", input_path.name, md_path.name, json_path.name)
+
+
 def _process_single(
     ocr_fn, input_path: Path,
     output_dir_resolved: Path, use_vlm: bool,
 ) -> None:
     """Обрабатывает одно изображение: OCR → fixups → корректоры → JSON + Markdown."""
-    # Этап OCR — самое долгое и хрупкое место
     logger.debug("OCR: начало %s", input_path.name)
     t0 = time.time()
     page = ocr_fn(str(input_path))
@@ -86,34 +118,15 @@ def _process_single(
     logger.info("Распознано %d блоков за %.1fs", len(page.blocks), dt)
     logger.debug("OCR: завершено %s (блоков: %d)", input_path.name, len(page.blocks))
 
-    # Исправление типичных OCR-ошибок (VaR, V^2)
     fix_ocr_errors(page.blocks)
     logger.debug("Fixups: завершено")
 
-    # Опциональная коррекция через VLM
+    _log_low_confidence(page.raw_json)
+
     if use_vlm:
-        from src.postprocess.corrector import apply_corrector
-        logger.debug("Корректоры: инициализация")
-        from src.postprocess.vlm_corrector import vlm_correct_block
-        corrector = functools.partial(vlm_correct_block, image_path=str(input_path.resolve()))
-        apply_corrector(page.blocks, corrector, "VLM")
-        logger.debug("VLM: завершено")
-
-    # Сохранение JSON (всегда) + Markdown (всегда)
-    json_path = output_dir_resolved / (input_path.stem + ".json")
-    if not json_path.exists():
-        # Если raw_json пуст (Surya), строим из ParsedBlock
-        json_payload = page.raw_json or {
-            "blocks": [{"label": b.label, "content": b.content, "bbox": list(b.bbox)}
-                       for b in page.blocks]
-        }
-        json_path.write_text(json.dumps(json_payload, ensure_ascii=False, indent=2), encoding="utf-8")
-        logger.debug("JSON сохранён: %s", json_path)
+        _apply_vlm(page.blocks, input_path)
 
-    md_path = output_dir_resolved / (input_path.stem + MD_EXTENSION)
-    generate_markdown(page.blocks, md_path)
-    logger.debug("Markdown сохранён: %s", md_path)
-    logger.info("%s → %s, %s", input_path.name, md_path.name, json_path.name)
+    _save_result(page, input_path, output_dir_resolved)
 
 
 @app.command()
@@ -180,7 +193,7 @@ def image_to_latex(
     ] = 5,
 ) -> None:
     """OCR → PostProcess → Markdown. Принимает файл или каталог изображений."""
-    ocr_fn, engine_name = _get_ocr_engine(ocr_engine, main_lang)
+    ocr_fn, engine_name, is_batch = _get_ocr_engine(ocr_engine, main_lang)
 
     if input.is_dir():
         image_paths = sorted(
@@ -214,18 +227,30 @@ def image_to_latex(
 
         total_start = time.time()
 
-        for i, img_path in enumerate(image_paths):
-            logger.info("[%d/%d] %s...", i + 1 + skipped, len(image_paths) + skipped, img_path.name)
-            _process_single(ocr_fn, img_path, out, use_vlm)
-            if i < len(image_paths) - 1 and pause > 0:
-                time.sleep(pause)
+        if is_batch:
+            paths = [str(p) for p in image_paths]
+            logger.info("Batch-обработка %d изображений...", len(paths))
+            pages = ocr_fn(paths)
+            for img_path, page in zip(image_paths, pages):
+                logger.info("%s → %d блоков", img_path.name, len(page.blocks))
+                fix_ocr_errors(page.blocks)
+                _log_low_confidence(page.raw_json)
+                if use_vlm:
+                    _apply_vlm(page.blocks, img_path)
+                _save_result(page, img_path, out)
+        else:
+            for i, img_path in enumerate(image_paths):
+                logger.info("[%d/%d] %s...", i + 1 + skipped, len(image_paths) + skipped, img_path.name)
+                _process_single(ocr_fn, img_path, out, use_vlm)
+                if i < len(image_paths) - 1 and pause > 0:
+                    time.sleep(pause)
 
         total_dt = time.time() - total_start
         processed = len(image_paths)
         avg_dt = total_dt / processed if processed else 0
 
         if result_one_document:
-            _merge_markdown_files(out, sorted(out.glob("*.md")))
+            _merge_markdown_files(out, input.name, sorted(out.glob("*.md")))
 
         logger.info(
             "Готово: %d изображений, всего %.1fs, среднее %.1fs/изобр",

src/image_prepare.py

@@ -140,7 +140,7 @@ def slice_pages(
             "--border",
             "-b",
             min=0,
-            help="Отступ в пикселях (белая рамка / отступ при обрезке содержимого, по умолчанию 5)",
+            help="Отступ в пикселях (белая рамка / отступ при обрезке содержимого, по умолчанию 10)",
         ),
     ] = DEFAULT_BORDER_PX,
     unwarp: Annotated[

src/ocr/surya_engine.py

@@ -72,6 +72,14 @@ class SuryaEngine:
         image = Image.open(image_path)
         return self._process_image(image)
 
+    def process_batch(self, image_paths: list[str]) -> list[OcrPageResult]:
+        from PIL import Image
+
+        images = [_pre_scale(Image.open(p)) for p in image_paths]
+        results = self._predictor(images)
+        return [self._to_result(results[i], images[i]) if i < len(results) else OcrPageResult()
+                for i in range(len(images))]
+
     def _health_check(self) -> bool:
         import requests
 
@@ -130,6 +138,11 @@ def _get_surya_engine() -> SuryaEngine:
 
 
 def surya_ocr_image(image_path: str) -> OcrPageResult:
+    engine = _get_surya_engine()
+    return engine.process(image_path)
+
+
+def surya_ocr_batch(image_paths: list[str]) -> list[OcrPageResult]:
     engine = _get_surya_engine()
     if not engine._health_check():
         logger.warning("llama-server не отвечает, перезапуск...")
@@ -141,4 +154,4 @@ def surya_ocr_image(image_path: str) -> OcrPageResult:
                 "Проверьте: `ps aux | grep llama-server`. "
                 "Попробуйте: `llama-server --version`"
             )
-    return engine.process(image_path)
+    return engine.process_batch(image_paths)

src/postprocess/vlm_corrector.py

@@ -26,9 +26,9 @@ def vlm_correct_block(block: ParsedBlock, image_path: str) -> None:
         return
 
     crop = image[y1:y2, x1:x2]
-    logger.info("VLM: отправка региона %dx%d px (bbox %d,%d,%d,%d)", x2 - x1, y2 - y1, x1, y1, x2, y2)
-
     b64 = image_to_base64(crop)
+    total_sent = len(b64) // 1024
+    logger.info("VLM: отправка %dx%d px → %d KB (bbox %d,%d,%d,%d)", x2 - x1, y2 - y1, total_sent, x1, y1, x2, y2)
     text = call_qwen_vlm(
         b64,
         "Extract ALL visible text from this image region. Preserve LaTeX math. Return ONLY the extracted text.",

src/split/slicer.py

@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
 
 SUPPORTED_EXTENSIONS: frozenset[str] = frozenset({".jpg", ".jpeg", ".png", ".tiff", ".tif"})
 PAGE_NUMBER_WIDTH: int = 2
-DEFAULT_BORDER_PX: int = 5
+DEFAULT_BORDER_PX: int = 10
 BORDER_COLOR: tuple[int, int, int] = (255, 255, 255)
 VALID_ROTATIONS: frozenset[int] = frozenset({90, 180, 270})
 DENSITY_RATIO: float = 0.005

tests/test_post_crop.py

@@ -121,7 +121,7 @@ class TestProcessImagePostCrop:
             result = process_image(Path(path), 1, 1, Path(tmp), post_crop=False)
             assert len(result) == 1
             loaded = cv2.imread(str(result[0]))
-            assert loaded.shape == (310, 410, 3)
+            assert loaded.shape == (320, 420, 3)
 
     def test_post_crop_e2e_2x2(self) -> None:
         image = create_test_image(400, 400)