ei
/
scan4llm
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

code refactoring

master
Evgeniy Ierusalimov 4 days ago
parent
25992f0ef6
commit
f25e5d3cf7
20 changed files with 330 additions and 257 deletions
Split View Show Diff Stats
  1. 8
    11
      README.md
  2. 8
    0
      src/image_utils.py
  3. 2
    2
      src/latex/__init__.py
  4. 248
    16
      src/latex/json_to_latex.py
  5. 0
    74
      src/latex/surya2html.py
  6. 0
    54
      src/latex/tex2html.py
  7. 1
    1
      src/markdown/generator.py
  8. 20
    0
      src/models.py
  9. 2
    1
      src/ocr/__init__.py
  10. 5
    5
      src/ocr/external_vlm_engine.py
  11. 6
    21
      src/ocr/paddle_engine.py
  12. 17
    17
      src/ocr/surya_engine.py
  13. 3
    7
      src/postprocess/corrector.py
  14. 1
    1
      src/postprocess/fixups.py
  15. 6
    6
      src/postprocess/llm_corrector.py
  16. 1
    11
      src/postprocess/qwen_client.py
  17. 1
    1
      src/postprocess/vlm_corrector.py
  18. 0
    20
      src/split/__init__.py
  19. 0
    7
      src/split/slicer.py
  20. 1
    2
      stage2_results.md

+ 8
- 11
README.md View File 

@@ -254,19 +254,19 @@ CLI (`image_ocr.py`) генерирует два файла для каждог
254 254
255 255 
 | Файл | Утилита | Назначение |
256 256 
 |------|---------|------------|
257 
-| `.html` | `src/latex/surya2html.py` | Surya JSON → HTML + MathJax |
258 
-| `.tex` | `src/latex/json_to_latex.py` | Paddle или Surya JSON → LaTeX (автодетект) |
257 
+| `.tex` | `src/latex/json_to_latex.py` | JSON (Paddle / Surya / Qwen) → LaTeX + HTML (автодетект) |
259 258
260 259 
 ```bash
261 
-# JSON → HTML (Surya)
262 
-python -c "from src.latex.surya2html import surya_json_to_html; \
263 
-  open('page.html','w').write(surya_json_to_html('page.json'))"
260 
+# JSON → LaTeX (один файл, автоопределение формата)
261 
+python -m src.latex.json_to_latex page.json
264 262
265 
-# JSON → LaTeX (Paddle или Surya — автоопределение)
263 
+# JSON → LaTeX + HTML (каталог — объединённый многостраничный)
264 
+python -m src.latex.json_to_latex cbr_en_2/
265 
+
266 
+# Программный вызов
266 267 
 python -c "from src.latex.json_to_latex import json_to_latex; \
267 268 
   open('page.tex','w').write(json_to_latex('page.json'))"
268 269
269 
-# Многостраничный LaTeX (все JSON в каталоге)
270 270 
 python -c "from src.latex.json_to_latex import multi_json_to_latex; \
271 271 
   multi_json_to_latex('out/', 'out/document.tex')"
272 272 
 ```
 
@@ -368,10 +368,7 @@ src/
368 368 
     vlm_corrector.py     — Qwen3.8 Max (API)
369 369 
   markdown/generator.py  — Markdown + валидация
370 370 
   latex/
371 
-    surya2html.py        — Surya JSON → HTML
372 
-    surya2latex.py       — Surya JSON → LaTeX
373 
-    json_to_latex.py     — Единый JSON→LaTeX (Paddle + Surya)
374 
-    tex2html.py           — TeX → HTML
371 
+    json_to_latex.py     — Единый JSON→LaTeX+HTML (Paddle + Surya + Qwen)
375 372 
 ```
376 373
377 374 
 ---

+ 8
- 0
src/image_utils.py View File 

@@ -0,0 +1,8 @@
1 
+from __future__ import annotations
2 
+
3 
+
4 
+def calc_scale_dims(w: int, h: int, max_w: int) -> tuple[int, int]:
5 
+    if w > max_w:
6 
+        ratio = max_w / w
7 
+        return (max_w, int(h * ratio))
8 
+    return (w, h)

+ 2
- 2
src/latex/__init__.py View File 

@@ -1,3 +1,3 @@
1 
-from src.latex.tex2html import tex_to_html
1 
+from src.latex.json_to_latex import json_to_html, json_to_latex, multi_json_to_latex
2 2
3 
-__all__ = ["tex_to_html"]
3 
+__all__ = ["json_to_html", "json_to_latex", "multi_json_to_latex"]

+ 248
- 16
src/latex/json_to_latex.py View File 

@@ -1,6 +1,8 @@
1 1 
 from __future__ import annotations
2 2
3 3 
 import json
4 
+import re
5 
+import sys
4 6 
 from pathlib import Path
5 7
6 8 
 from bs4 import BeautifulSoup
 
@@ -13,8 +15,93 @@ LATEX_PREAMBLE = r"""\documentclass[12pt,a4paper]{article}
13 15 
 \usepackage{geometry}
14 16 
 \usepackage{booktabs}
15 17 
 \geometry{margin=2cm}
16 
-\begin{document}
17 18 
 """
19 
+LATEX_BEGIN = "\\begin{document}\n"
20 
+LATEX_END = "\n\\end{document}\n"
21 
+
22 
+HTML_TEMPLATE = """\
23 
+<!DOCTYPE html>
24 
+<html lang="en">
25 
+<head>
26 
+<meta charset="utf-8">
27 
+<title>{title}</title>
28 
+<script>
29 
+MathJax = {{
30 
+  tex: {{
31 
+    inlineMath: [['$', '$']],
32 
+    displayMath: [['$$', '$$']],
33 
+    packages: {{'[+]': ['ams']}}
34 
+  }},
35 
+  loader: {{load: ['[tex]/ams']}}
36 
+}};
37 
+</script>
38 
+<script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js" async></script>
39 
+<style>
40 
+  body {{ font-family: serif; max-width: 800px; margin: 2em auto; padding: 0 1em; line-height: 1.6; }}
41 
+  h2 {{ margin-top: 1.5em; }}
42 
+  .page-sep {{ text-align: center; color: #888; margin: 2em 0; font-size: 0.9em; }}
43 
+  hr {{ border: none; border-top: 1px solid #ccc; }}
44 
+  table {{ border-collapse: collapse; margin: 1em 0; width: 100%; }}
45 
+  th, td {{ border: 1px solid #aaa; padding: 4px 8px; text-align: center; }}
46 
+  th {{ background: #eaeaea; }}
47 
+  .equation {{ text-align: center; margin: 1em 0; }}
48 
+</style>
49 
+</head>
50 
+<body>
51 
+{body}
52 
+</body>
53 
+</html>
54 
+"""
55 
+
56 
+MD_HEADING: re.Pattern = re.compile(r"^#{1,6}\s+")
57 
+MD_TABLE_SEP: re.Pattern = re.compile(r"^\|?[\s:-]+\|[\s|:-]+\|?$")
58 
+
59 
+
60 
+def _is_md_table(text: str) -> bool:
61 
+    lines = text.strip().splitlines()
62 
+    if len(lines) < 3:
63 
+        return False
64 
+    has_sep = any(MD_TABLE_SEP.match(line) for line in lines)
65 
+    if not has_sep:
66 
+        return False
67 
+    return all("|" in line for line in lines)
68 
+
69 
+
70 
+def _parse_md_table(text: str) -> list[list[str]]:
71 
+    lines = text.strip().splitlines()
72 
+    rows: list[list[str]] = []
73 
+    for line in lines:
74 
+        if MD_TABLE_SEP.match(line):
75 
+            continue
76 
+        cells = [c.strip() for c in line.split("|")[1:-1]]
77 
+        rows.append(cells)
78 
+    return rows
79 
+
80 
+
81 
+def _md_table_to_html(text: str) -> str:
82 
+    rows = _parse_md_table(text)
83 
+    if not rows:
84 
+        return ""
85 
+    html_rows: list[str] = []
86 
+    for i, row in enumerate(rows):
87 
+        tag = "th" if i == 0 or not html_rows else "td"
88 
+        cells = "".join(f"<{tag}>{c}</{tag}>" for c in row)
89 
+        html_rows.append(f"<tr>{cells}</tr>")
90 
+    return "<table>\n" + "\n".join(html_rows) + "\n</table>"
91 
+
92 
+
93 
+def _md_table_to_latex(text: str) -> str:
94 
+    rows = _parse_md_table(text)
95 
+    if not rows:
96 
+        return ""
97 
+    ncols = max(len(row) for row in rows)
98 
+    lines = [f"\\begin{{tabular}}{{|{'c|' * ncols}}}", "\\hline"]
99 
+    for row in rows:
100 
+        padded = row + [""] * (ncols - len(row))
101 
+        lines.append(" & ".join(padded) + " \\\\")
102 
+        lines.append("\\hline")
103 
+    lines.append("\\end{tabular}")
104 
+    return "\n".join(lines)
18 105
19 106
20 107 
 def _detect_format(data: dict) -> str:
 
@@ -22,16 +109,20 @@ def _detect_format(data: dict) -> str:
22 109 
         first = data["blocks"][0] if data["blocks"] else {}
23 110 
         if "html" in first:
24 111 
             return "surya"
112 
+        if "content" in first:
113 
+            return "qwen"
25 114 
     if "parsing_res_list" in data:
26 115 
         return "paddle"
27 
-    raise ValueError("Неизвестный формат JSON: ожидается PaddleOCR или Surya")
116 
+    raise ValueError("Неизвестный формат JSON: ожидается PaddleOCR, Surya или Qwen")
28 117
29 118
30 
-def _escape(text: str) -> str:
31 
-    for char, repl in [("\\", "\\textbackslash{}"), ("&", "\\&"), ("%", "\\%"),
32 
-                       ("$", "\\$"), ("#", "\\#"), ("_", "\\_"),
33 
-                       ("{", "\\{"), ("}", "\\}"), ("~", "\\textasciitilde{}"),
34 
-                       ("^", "\\^{}"), ("<", "\\textless{}"), (">", "\\textgreater{}")]:
119 
+def _escape_latex(text: str) -> str:
120 
+    for char, repl in [
121 
+        ("\\", "\\textbackslash{}"), ("&", "\\&"), ("%", "\\%"),
122 
+        ("$", "\\$"), ("#", "\\#"), ("_", "\\_"),
123 
+        ("{", "\\{"), ("}", "\\}"), ("~", "\\textasciitilde{}"),
124 
+        ("^", "\\^{}"), ("<", "\\textless{}"), (">", "\\textgreater{}"),
125 
+    ]:
35 126 
         text = text.replace(char, repl)
36 127 
     return text
37 128 
 
@@ -47,11 +138,11 @@ def _paddle_to_body(data: dict) -> str:
47 138 
         if label == "formula":
48 139 
             lines.append(f"\\[\n{content}\n\\]\n")
49 140 
         elif label in ("paragraph_title", "title", "section_header"):
50 
-            lines.append(f"\\section*{{{_escape(content)}}}")
141 
+            lines.append(f"\\section*{{{_escape_latex(content)}}}")
51 142 
         elif label == "text":
52 
-            lines.append(f"\n{_escape(content)}\n")
143 
+            lines.append(f"\n{_escape_latex(content)}\n")
53 144 
         else:
54 
-            lines.append(f"\n{_escape(content)}\n")
145 
+            lines.append(f"\n{_escape_latex(content)}\n")
55 146 
     return "\n".join(lines)
56 147
57 148 
 
@@ -93,6 +184,30 @@ def _surya_to_body(data: dict) -> str:
93 184 
     return "\n".join(lines)
94 185
95 186
187 
+def _qwen_to_body(data: dict) -> str:
188 
+    lines: list[str] = []
189 
+    for b in data.get("blocks", []):
190 
+        content = b.get("content", "").strip()
191 
+        if not content:
192 
+            continue
193 
+        label = b.get("label", "text")
194 
+
195 
+        if label == "formula":
196 
+            formula = content.strip()
197 
+            if (formula.startswith("$$") and formula.endswith("$$")) or \
198 
+               (formula.startswith("\\[") and formula.endswith("\\]")):
199 
+                formula = formula[2:-2].strip()
200 
+            lines.append(f"\\[\n{formula}\n\\]\n")
201 
+        elif label == "SectionHeader":
202 
+            heading = MD_HEADING.sub("", content).strip()
203 
+            lines.append(f"\\section*{{{_escape_latex(heading)}}}")
204 
+        elif _is_md_table(content):
205 
+            lines.append(f"\n{_md_table_to_latex(content)}\n")
206 
+        else:
207 
+            lines.append(f"\n{content}\n")
208 
+    return "\n".join(lines)
209 
+
210 
+
96 211 
 def _table_to_latex(soup: BeautifulSoup) -> str:
97 212 
     rows = soup.find_all("tr")
98 213 
     if not rows:
 
@@ -113,10 +228,12 @@ def json_to_latex(json_path: str | Path) -> str:
113 228
114 229 
     if fmt == "paddle":
115 230 
         body = _paddle_to_body(data)
116 
-    else:
231 
+    elif fmt == "surya":
117 232 
         body = _surya_to_body(data)
233 
+    else:
234 
+        body = _qwen_to_body(data)
118 235
119 
-    return LATEX_PREAMBLE + body + "\n\\end{document}\n"
236 
+    return LATEX_PREAMBLE + LATEX_BEGIN + body + LATEX_END
120 237
121 238
122 239 
 def multi_json_to_latex(json_dir: str | Path, output_path: str | Path | None = None) -> str:
 
@@ -127,8 +244,8 @@ def multi_json_to_latex(json_dir: str | Path, output_path: str | Path | None = N
127 244 
             continue
128 245 
         try:
129 246 
             body = json_to_latex(jp)
130 
-            start = body.find("\\begin{document}") + len("\\begin{document}")
131 
-            end = body.find("\\end{document}")
247 
+            start = body.find(LATEX_BEGIN) + len(LATEX_BEGIN)
248 
+            end = body.find(LATEX_END)
132 249 
             pages.append(f"% {jp.stem}\n{body[start:end].strip()}\n\\newpage")
133 250 
         except (json.JSONDecodeError, KeyError, ValueError):
134 251 
             continue
 
@@ -138,9 +255,124 @@ def multi_json_to_latex(json_dir: str | Path, output_path: str | Path | None = N
138 255
139 256 
     first_path = min(json_dir.glob("[!.]*.json"), key=lambda p: p.stem)
140 257 
     first = json_to_latex(first_path)
141 
-    preamble_end = first.find("\\begin{document}") + len("\\begin{document}")
142 
-    full = first[:preamble_end] + "\n" + "\n".join(pages) + "\n\\end{document}\n"
258 
+    preamble_end = first.find(LATEX_BEGIN) + len(LATEX_BEGIN)
259 
+    full = first[:preamble_end] + "\n" + "\n".join(pages) + LATEX_END
143 260
144 261 
     if output_path:
145 262 
         Path(output_path).write_text(full, encoding="utf-8")
146 263 
     return full
264 
+
265 
+
266 
+def _json_block_to_html(block: dict, fmt: str) -> str:
267 
+    if fmt == "qwen":
268 
+        label = block.get("label", "text")
269 
+        content = block.get("content", "").strip()
270 
+        if not content:
271 
+            return ""
272 
+        if label == "formula":
273 
+            clean = content.strip()
274 
+            for prefix, suffix in [("$$", "$$"), ("\\[", "\\]")]:
275 
+                if clean.startswith(prefix) and clean.endswith(suffix):
276 
+                    clean = clean[len(prefix):-len(suffix)].strip()
277 
+            return f'\n<div class="equation">$$\n{clean}\n$$</div>'
278 
+        if label == "SectionHeader":
279 
+            heading = MD_HEADING.sub("", content).strip()
280 
+            return f"<h2>{heading}</h2>"
281 
+        if _is_md_table(content):
282 
+            return _md_table_to_html(content)
283 
+        return f"<p>{content}</p>"
284 
+
285 
+    if fmt == "paddle":
286 
+        content = block.get("block_content", "").strip()
287 
+        if not content:
288 
+            return ""
289 
+        label = block.get("block_label", "text")
290 
+        if label == "formula":
291 
+            return f'\n<div class="equation">$$\n{content}\n$$</div>'
292 
+        if label in ("paragraph_title", "title", "section_header"):
293 
+            return f"<h2>{content}</h2>"
294 
+        return f"<p>{content}</p>"
295 
+
296 
+    if fmt == "surya":
297 
+        label = block.get("label", "")
298 
+        raw_html = block.get("html", "").strip()
299 
+        if not raw_html:
300 
+            return ""
301 
+        if label == "Equation":
302 
+            clean = raw_html.replace('<math display="block">', "").replace("</math>", "").strip()
303 
+            return f'\n<div class="equation">$$\n{clean}\n$$</div>'
304 
+        soup = BeautifulSoup(raw_html, "html.parser")
305 
+        for math_tag in soup.find_all("math"):
306 
+            display = math_tag.get("display", "") == "block"
307 
+            latex = math_tag.get_text().strip()
308 
+            math_tag.replace_with(f"$$\n{latex}\n$$" if display else f"${latex}$")
309 
+        text = " ".join(soup.stripped_strings)
310 
+        if not text:
311 
+            return ""
312 
+        if label in ("SectionHeader",):
313 
+            return f"<h2>{text}</h2>"
314 
+        if label == "PageHeader":
315 
+            return f'<p class="header">{text}</p>'
316 
+        return f"<p>{text}</p>"
317 
+    return ""
318 
+
319 
+
320 
+def json_to_html(json_path: str | Path) -> str:
321 
+    data = json.loads(Path(json_path).read_text())
322 
+    fmt = _detect_format(data)
323 
+    title = Path(json_path).stem
324 
+
325 
+    blocks = data.get("blocks", data.get("parsing_res_list", []))
326 
+    html_parts = [_json_block_to_html(b, fmt) for b in blocks]
327 
+    body = "\n".join(p for p in html_parts if p)
328 
+
329 
+    return HTML_TEMPLATE.format(title=title, body=body)
330 
+
331 
+
332 
+def _generate_all(json_dir: Path) -> None:
333 
+    dirname = json_dir.name or json_dir.parent.name
334 
+    json_dir = json_dir.resolve()
335 
+    tex_path = json_dir / f"{dirname}.tex"
336 
+    html_path = json_dir / f"{dirname}.html"
337 
+
338 
+    _ = multi_json_to_latex(json_dir, tex_path)
339 
+    print(f"LaTeX: {tex_path}")
340 
+
341 
+    pages: list[str] = []
342 
+    for jp in sorted(json_dir.glob("[!.]*.json")):
343 
+        if jp.stem == dirname:
344 
+            continue
345 
+        try:
346 
+            page_html = json_to_html(jp)
347 
+            inner = page_html[page_html.find("<body>") + 6:page_html.find("</body>")]
348 
+            pages.append(f'<div class="page-sep">{jp.stem}</div>\n<hr>\n{inner}')
349 
+        except (json.JSONDecodeError, KeyError, ValueError):
350 
+            continue
351 
+
352 
+    if pages:
353 
+        merged = HTML_TEMPLATE.format(title=dirname, body="\n".join(pages))
354 
+        html_path.write_text(merged, encoding="utf-8")
355 
+        print(f"HTML:   {html_path}")
356 
+
357 
+
358 
+def main() -> None:
359 
+    if len(sys.argv) < 2:
360 
+        print("Usage: python -m src.latex.json_to_latex <file.json | dir/>", file=sys.stderr)
361 
+        sys.exit(1)
362 
+
363 
+    input_path = Path(sys.argv[1]).resolve()
364 
+
365 
+    if input_path.is_dir():
366 
+        _generate_all(input_path)
367 
+    else:
368 
+        output = input_path.with_suffix(".tex")
369 
+        output.write_text(json_to_latex(input_path), encoding="utf-8")
370 
+        print(f"LaTeX: {output}")
371 
+
372 
+        html_out = input_path.with_suffix(".html")
373 
+        html_out.write_text(json_to_html(input_path), encoding="utf-8")
374 
+        print(f"HTML:   {html_out}")
375 
+
376 
+
377 
+if __name__ == "__main__":
378 
+    main()

+ 0
- 74
src/latex/surya2html.py View File 

@@ -1,74 +0,0 @@
1 
-from __future__ import annotations
2 
-
3 
-import json
4 
-from pathlib import Path
5 
-
6 
-from bs4 import BeautifulSoup
7 
-
8 
-HTML_TEMPLATE = """\
9 
-<!DOCTYPE html>
10 
-<html lang="en">
11 
-<head>
12 
-<meta charset="utf-8">
13 
-<title>{title} — Surya 2</title>
14 
-<script>
15 
-MathJax = {{
16 
-  tex: {{
17 
-    inlineMath: [['$', '$']],
18 
-    displayMath: [['$$', '$$']],
19 
-    packages: {{'[+]': ['ams']}}
20 
-  }},
21 
-  loader: {{load: ['[tex]/ams']}}
22 
-}};
23 
-</script>
24 
-<script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js" async></script>
25 
-<style>
26 
-  body {{ font-family: serif; max-width: 800px; margin: 2em auto; padding: 0 1em; line-height: 1.6; }}
27 
-  h2 {{ margin-top: 1.5em; }}
28 
-  .equation {{ text-align: center; margin: 1em 0; }}
29 
-  .header {{ font-weight: bold; }}
30 
-  p {{ margin: 0.5em 0; text-indent: 1.5em; }}
31 
-</style>
32 
-</head>
33 
-<body>
34 
-{body}
35 
-</body>
36 
-</html>
37 
-"""
38 
-
39 
-
40 
-def surya_json_to_html(json_path: str | Path) -> str:
41 
-    data = json.loads(Path(json_path).read_text())
42 
-    blocks = data.get("blocks", [])
43 
-
44 
-    html_parts: list[str] = []
45 
-    for b in blocks:
46 
-        label = b.get("label", "")
47 
-        raw_html = b.get("html", "")
48 
-        if not raw_html.strip():
49 
-            continue
50 
-
51 
-        if label == "Equation":
52 
-            clean = raw_html.replace('<math display="block">', "").replace("</math>", "").strip()
53 
-            html_parts.append(f'\n<div class="equation">$$\n{clean}\n$$</div>')
54 
-            continue
55 
-
56 
-        soup = BeautifulSoup(raw_html, "html.parser")
57 
-        for math_tag in soup.find_all("math"):
58 
-            display = math_tag.get("display", "") == "block"
59 
-            latex = math_tag.get_text().strip()
60 
-            math_tag.replace_with(f"$$\n{latex}\n$$" if display else f"${latex}$")
61 
-
62 
-        text = " ".join(soup.stripped_strings)
63 
-        if not text:
64 
-            continue
65 
-
66 
-        if label in ("SectionHeader",):
67 
-            html_parts.append(f"<h2>{text}</h2>")
68 
-        elif label == "PageHeader":
69 
-            html_parts.append(f'<p class="header">{text}</p>')
70 
-        else:
71 
-            html_parts.append(f"<p>{text}</p>")
72 
-
73 
-    body = "\n".join(html_parts)
74 
-    return HTML_TEMPLATE.format(title=Path(json_path).stem, body=body)

+ 0
- 54
src/latex/tex2html.py View File 

@@ -1,54 +0,0 @@
1 
-from __future__ import annotations
2 
-
3 
-import sys
4 
-from pathlib import Path
5 
-
6 
-HTML_TEMPLATE = """\
7 
-<!DOCTYPE html>
8 
-<html lang="ru">
9 
-<head>
10 
-<meta charset="utf-8">
11 
-<title>{title}</title>
12 
-<script>
13 
-MathJax = {{
14 
-  tex: {{
15 
-    inlineMath: [['$', '$']],
16 
-    displayMath: [['$$', '$$']],
17 
-    processEscapes: true
18 
-  }}
19 
-}};
20 
-</script>
21 
-<script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js" async></script>
22 
-<style>
23 
-  body {{ font-family: serif; max-width: 800px; margin: 2em auto; padding: 0 1em; line-height: 1.6; }}
24 
-  p {{ text-indent: 1.5em; margin: 0.5em 0; }}
25 
-</style>
26 
-</head>
27 
-<body>
28 
-<pre style="white-space: pre-wrap; font-family: serif;">
29 
-{body}
30 
-</pre>
31 
-</body>
32 
-</html>
33 
-"""
34 
-
35 
-
36 
-def tex_to_html(tex_path: Path) -> Path:
37 
-    tex_content = tex_path.read_text(encoding="utf-8")
38 
-    start = tex_content.find("\\begin{document}") + len("\\begin{document}")
39 
-    end = tex_content.find("\\end{document}")
40 
-    body = tex_content[start:end].strip()
41 
-
42 
-    html_content = HTML_TEMPLATE.format(
43 
-        title=tex_path.stem,
44 
-        body=body,
45 
-    )
46 
-    html_path = tex_path.with_suffix(".html")
47 
-    html_path.write_text(html_content, encoding="utf-8")
48 
-    return html_path
49 
-
50 
-
51 
-if __name__ == "__main__":
52 
-    for p in sys.argv[1:]:
53 
-        result = tex_to_html(Path(p))
54 
-        print(f"HTML: {result}")

+ 1
- 1
src/markdown/generator.py View File 

@@ -4,7 +4,7 @@ import logging
4 4 
 import re
5 5 
 from pathlib import Path
6 6
7 
-from src.ocr.paddle_engine import ParsedBlock
7 
+from src.models import ParsedBlock
8 8
9 9 
 logger = logging.getLogger(__name__)
10 10

+ 20
- 0
src/models.py View File 

@@ -0,0 +1,20 @@
1 
+from __future__ import annotations
2 
+
3 
+from dataclasses import dataclass, field
4 
+from typing import Any
5 
+
6 
+
7 
+@dataclass
8 
+class ParsedBlock:
9 
+    label: str
10 
+    content: str
11 
+    bbox: tuple[int, int, int, int]
12 
+    confidence: float = 0.9
13 
+
14 
+
15 
+@dataclass
16 
+class OcrPageResult:
17 
+    blocks: list[ParsedBlock] = field(default_factory=list)
18 
+    raw_json: dict[str, Any] = field(default_factory=dict)
19 
+    width: int = 0
20 
+    height: int = 0

+ 2
- 1
src/ocr/__init__.py View File 

@@ -1,3 +1,4 @@
1 
-from src.ocr.paddle_engine import OcrPageResult, ParsedBlock, ocr_image
1 
+from src.models import OcrPageResult, ParsedBlock
2 
+from src.ocr.paddle_engine import ocr_image
2 3
3 4 
 __all__ = ["OcrPageResult", "ParsedBlock", "ocr_image"]

+ 5
- 5
src/ocr/external_vlm_engine.py View File 

@@ -5,7 +5,8 @@ import time
5 5
6 6 
 import cv2
7 7
8 
-from src.ocr.paddle_engine import OcrPageResult, ParsedBlock
8 
+from src.image_utils import calc_scale_dims
9 
+from src.models import OcrPageResult, ParsedBlock
9 10 
 from src.postprocess.qwen_client import call_qwen_vlm, image_to_base64
10 11
11 12 
 logger = logging.getLogger(__name__)
 
@@ -14,10 +15,9 @@ _IMAGE_MAX_WIDTH: int = 1024
14 15
15 16
16 17 
 def _pre_scale(image):
17 
-    h, w = image.shape[:2]
18 
-    if w > _IMAGE_MAX_WIDTH:
19 
-        ratio = _IMAGE_MAX_WIDTH / w
20 
-        return cv2.resize(image, (_IMAGE_MAX_WIDTH, int(h * ratio)))
18 
+    new_w, new_h = calc_scale_dims(image.shape[1], image.shape[0], _IMAGE_MAX_WIDTH)
19 
+    if (new_w, new_h) != (image.shape[1], image.shape[0]):
20 
+        return cv2.resize(image, (new_w, new_h))
21 21 
     return image
22 22
23 23

+ 6
- 21
src/ocr/paddle_engine.py View File 

@@ -1,28 +1,14 @@
1 1 
 from __future__ import annotations
2 2
3 
+import functools
3 4 
 import importlib.resources
4 
-from dataclasses import dataclass, field
5 5 
 from typing import Any
6 6
7 7 
 import cv2
8 8 
 import yaml
9 9 
 from paddlex import create_pipeline
10 10
11 
-
12 
-@dataclass
13 
-class ParsedBlock:
14 
-    label: str
15 
-    content: str
16 
-    bbox: tuple[int, int, int, int]
17 
-    confidence: float = 0.9
18 
-
19 
-
20 
-@dataclass
21 
-class OcrPageResult:
22 
-    blocks: list[ParsedBlock] = field(default_factory=list)
23 
-    raw_json: dict[str, Any] = field(default_factory=dict)
24 
-    width: int = 0
25 
-    height: int = 0
11 
+from src.models import OcrPageResult, ParsedBlock
26 12
27 13
28 14 
 def _load_config() -> dict[str, Any]:
 
@@ -80,11 +66,10 @@ class OcrEngine:
80 66 
         return blocks, raw_json
81 67
82 68
83 
-_engine: OcrEngine | None = None
69 
+@functools.lru_cache(maxsize=1)
70 
+def _get_engine() -> OcrEngine:
71 
+    return OcrEngine()
84 72
85 73
86 74 
 def ocr_image(image_path: str) -> OcrPageResult:
87 
-    global _engine
88 
-    if _engine is None:
89 
-        _engine = OcrEngine()
90 
-    return _engine.process(image_path)
75 
+    return _get_engine().process(image_path)

+ 17
- 17
src/ocr/surya_engine.py View File 

@@ -1,5 +1,6 @@
1 1 
 from __future__ import annotations
2 2
3 
+import functools
3 4 
 import logging
4 5 
 import os
5 6 
 import shutil
 
@@ -7,7 +8,8 @@ from pathlib import Path
7 8 
 from typing import TYPE_CHECKING
8 9
9 10 
 from src.env import load_env
10 
-from src.ocr.paddle_engine import OcrPageResult, ParsedBlock
11 
+from src.image_utils import calc_scale_dims
12 
+from src.models import OcrPageResult, ParsedBlock
11 13
12 14 
 if TYPE_CHECKING:
13 15 
     from PIL import Image as PILImage
 
@@ -20,7 +22,7 @@ _SURYA_PARALLEL: int = 1
20 22 
 _IMAGE_MAX_WIDTH: int = 1056
21 23
22 24
23 
-def _ensure_env() -> None:
25 
+def _setup_surya_env() -> None:
24 26 
     load_env()
25 27 
     os.environ.setdefault("SURYA_INFERENCE_BACKEND", "llamacpp")
26 28 
     os.environ.setdefault("SURYA_INFERENCE_KEEP_ALIVE", "true")
 
@@ -51,7 +53,7 @@ class SuryaEngine:
51 53 
     """
52 54
53 55 
     def __init__(self) -> None:
54 
-        _ensure_env()
56 
+        _setup_surya_env()
55 57 
         try:
56 58 
             from surya.inference import SuryaInferenceManager
57 59 
             from surya.recognition import RecognitionPredictor
 
@@ -116,29 +118,27 @@ class SuryaEngine:
116 118
117 119
118 120 
 def _pre_scale(image: PILImage.Image) -> PILImage.Image:
119 
-    w, h = image.size
120 
-    if w > _IMAGE_MAX_WIDTH:
121 
-        ratio = _IMAGE_MAX_WIDTH / w
122 
-        return image.resize((_IMAGE_MAX_WIDTH, int(h * ratio)), 1)  # PIL.Image.LANCZOS
121 
+    new_w, new_h = calc_scale_dims(*image.size, _IMAGE_MAX_WIDTH)
122 
+    if (new_w, new_h) != image.size:
123 
+        return image.resize((new_w, new_h), 1)  # PIL.Image.LANCZOS
123 124 
     return image
124 125
125 126
126 
-_surya_engine: SuryaEngine | None = None
127 
+@functools.lru_cache(maxsize=1)
128 
+def _get_surya_engine() -> SuryaEngine:
129 
+    return SuryaEngine()
127 130
128 131
129 132 
 def surya_ocr_image(image_path: str) -> OcrPageResult:
130 
-    global _surya_engine
131 
-    if _surya_engine is None:
132 
-        _surya_engine = SuryaEngine()
133 
-
134 
-    if not _surya_engine._health_check():
133 
+    engine = _get_surya_engine()
134 
+    if not engine._health_check():
135 135 
         logger.warning("llama-server не отвечает, перезапуск...")
136 
-        _surya_engine = SuryaEngine()  # пересоздать — заново spawn сервер
137 
-        if not _surya_engine._health_check():
136 
+        _get_surya_engine.cache_clear()
137 
+        engine = _get_surya_engine()
138 
+        if not engine._health_check():
138 139 
             raise RuntimeError(
139 140 
                 "llama-server недоступен после перезапуска. "
140 141 
                 "Проверьте: `ps aux | grep llama-server`. "
141 142 
                 "Попробуйте: `llama-server --version`"
142 143 
             )
143 
-
144 
-    return _surya_engine.process(image_path)
144 
+    return engine.process(image_path)

+ 3
- 7
src/postprocess/corrector.py View File 

@@ -1,20 +1,16 @@
1 1 
 from __future__ import annotations
2 2
3 3 
 import logging
4 
-from typing import Protocol
4 
+from collections.abc import Callable
5 5
6 
-from src.ocr.paddle_engine import ParsedBlock
6 
+from src.models import ParsedBlock
7 7
8 8 
 logger = logging.getLogger(__name__)
9 9
10 10
11 
-class Corrector(Protocol):
12 
-    def __call__(self, block: ParsedBlock) -> None: ...
13 
-
14 
-
15 11 
 def apply_corrector(
16 12 
     blocks: list[ParsedBlock],
17 
-    corrector: Corrector,
13 
+    corrector: Callable[[ParsedBlock], None],
18 14 
     name: str,
19 15 
     skip_formulas: bool = True,
20 16 
 ) -> None:

+ 1
- 1
src/postprocess/fixups.py View File 

@@ -2,7 +2,7 @@ from __future__ import annotations
2 2
3 3 
 import re
4 4
5 
-from src.ocr.paddle_engine import ParsedBlock
5 
+from src.models import ParsedBlock
6 6
7 7 
 _SPACED_VARS = re.compile(r"\b([A-Z])\s+([A-Z])\b")
8 8

+ 6
- 6
src/postprocess/llm_corrector.py View File 

@@ -1,10 +1,11 @@
1 1 
 from __future__ import annotations
2 2
3 
+import functools
3 4 
 import os
4 5
5 6 
 from llama_cpp import Llama
6 7
7 
-from src.ocr.paddle_engine import ParsedBlock
8 
+from src.models import ParsedBlock
8 9
9 10 
 DEFAULT_LLM_PATH = os.path.expanduser("~/.cache/llama_models/Qwen2.5-7B-Instruct-Q4_K_M.gguf")
10 11 
 
@@ -34,11 +35,10 @@ class LlmCorrector:
34 35 
             block.content = corrected
35 36
36 37
37 
-_llm_corrector: LlmCorrector | None = None
38 
+@functools.lru_cache(maxsize=1)
39 
+def _get_llm_corrector() -> LlmCorrector:
40 
+    return LlmCorrector()
38 41
39 42
40 43 
 def llm_correct_block(block: ParsedBlock) -> None:
41 
-    global _llm_corrector
42 
-    if _llm_corrector is None:
43 
-        _llm_corrector = LlmCorrector()
44 
-    _llm_corrector(block)
44 
+    _get_llm_corrector()(block)

+ 1
- 11
src/postprocess/qwen_client.py View File 

@@ -16,19 +16,9 @@ API_URL: str = "https://opencode.ai/zen/go/v1/messages"
16 16 
 DEFAULT_MODEL: str = "qwen3.8-max"
17 17 
 IMAGE_QUALITY: int = 90
18 18
19 
-_ENV_LOADED: bool = False
20 
-
21 
-
22 
-def _ensure_env() -> None:
23 
-    global _ENV_LOADED
24 
-    if _ENV_LOADED:
25 
-        return
26 
-    load_env()
27 
-    _ENV_LOADED = True
28 
-
29 19
30 20 
 def _api_key() -> str:
31 
-    _ensure_env()
21 
+    load_env()
32 22 
     return os.environ.get("OPENCODE_API_KEY", "")
33 23
34 24

+ 1
- 1
src/postprocess/vlm_corrector.py View File 

@@ -4,7 +4,7 @@ import logging
4 4
5 5 
 import cv2
6 6
7 
-from src.ocr.paddle_engine import ParsedBlock
7 
+from src.models import ParsedBlock
8 8 
 from src.postprocess.qwen_client import call_qwen_vlm, image_to_base64
9 9
10 10 
 logger = logging.getLogger(__name__)

+ 0
- 20
src/split/__init__.py View File 

@@ -1,39 +1,19 @@
1 1 
 from src.split.slicer import (
2 2 
     DEFAULT_BORDER_PX,
3 
-    PAGE_NUMBER_WIDTH,
4 
-    SUPPORTED_EXTENSIONS,
5 3 
     VALID_ROTATIONS,
6 
-    add_border,
7 
-    crop_to_content,
8 4 
     detect_grid,
9 
-    find_content_bounds,
10 
-    generate_output_paths,
11 
-    is_empty,
12 5 
     load_image,
13 6 
     parse_slice,
14 7 
     process_image,
15 8 
     rotate_image,
16 
-    save_page,
17 
-    slice_grid,
18 
-    validate_output_dir,
19 9 
 )
20 10
21 11 
 __all__ = [
22 12 
     "DEFAULT_BORDER_PX",
23 
-    "PAGE_NUMBER_WIDTH",
24 
-    "SUPPORTED_EXTENSIONS",
25 13 
     "VALID_ROTATIONS",
26 
-    "add_border",
27 
-    "crop_to_content",
28 14 
     "detect_grid",
29 
-    "find_content_bounds",
30 
-    "generate_output_paths",
31 
-    "is_empty",
32 15 
     "load_image",
33 16 
     "parse_slice",
34 17 
     "process_image",
35 18 
     "rotate_image",
36 
-    "save_page",
37 
-    "slice_grid",
38 
-    "validate_output_dir",
39 19 
 ]

+ 0
- 7
src/split/slicer.py View File 

@@ -113,13 +113,6 @@ def parse_slice(value: str) -> tuple[int, int]:
113 113 
     return cols, rows
114 114
115 115
116 
-def validate_output_dir(path: Path) -> Path:
117 
-    path.mkdir(parents=True, exist_ok=True)
118 
-    if not path.is_dir():
119 
-        raise NotADirectoryError(f"Не удалось создать каталог: {path}")
120 
-    return path
121 
-
122 
-
123 116 
 def is_empty(image: np.ndarray) -> bool:
124 117 
     gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
125 118 
     dark_pixels = (gray < EMPTY_DARK_THRESHOLD).sum()

+ 1
- 2
stage2_results.md View File 

@@ -137,8 +137,7 @@ src/
137 137 
   markdown/generator.py  — Markdown + валидация LaTeX
138 138 
   latex/
139 139 
     json_to_latex.py     — Единый JSON→LaTeX (Paddle + Surya)
140 
-    surya2html.py        — Surya JSON → HTML + MathJax
141 
-    tex2html.py          — TeX → HTML
140 
+    json_to_latex.py     — Единый JSON→LaTeX+HTML (Paddle + Surya + Qwen)
142 141 
 ```
143 142
144 143 
 ---

Write Preview
Loading…
Cancel
Save