From 71676dba9e2520b487dd279f3642fe2705dc84e4 Mon Sep 17 00:00:00 2001 From: plf1996 Date: Thu, 18 Jun 2026 17:00:40 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20.txt=20=E7=BA=AF?= =?UTF-8?q?=E6=96=87=E6=9C=AC=E6=96=87=E4=BB=B6=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 添加 txt 到允许上传格式，跳过 MinerU OCR - 直接读取 UTF-8 文本，按 3000 字符/页分页后送入 LangExtract - 适合有中文文本的规范文件，避免扫描件 OCR 乱码问题 --- backend/routers/system.py | 1 + backend/services/document_service.py | 2 +- backend/services/indexing_service.py | 130 +++++++++++++++++---------- 3 files changed, 84 insertions(+), 49 deletions(-) diff --git a/backend/routers/system.py b/backend/routers/system.py index 750b84e..35a81e2 100644 --- a/backend/routers/system.py +++ b/backend/routers/system.py @@ -110,6 +110,7 @@ async def list_formats(): {"ext": "jpg", "description": "JPEG 图片（单页）", "max_size_mb": 200, "max_pages": 1, "requires_ocr": True}, {"ext": "jpeg", "description": "JPEG 图片（单页）", "max_size_mb": 200, "max_pages": 1, "requires_ocr": True}, {"ext": "html", "description": "HTML 文件", "max_size_mb": 200, "max_pages": 600, "requires_ocr": False}, + {"ext": "txt", "description": "纯文本文件（UTF-8 编码）", "max_size_mb": 200, "max_pages": 0, "requires_ocr": False}, ], "ocr_languages": [ {"code": "ch", "name": "中文（默认）"}, diff --git a/backend/services/document_service.py b/backend/services/document_service.py index cc8c91d..0193329 100644 --- a/backend/services/document_service.py +++ b/backend/services/document_service.py @@ -7,7 +7,7 @@ from pathlib import Path from storage import file_store as fs -ALLOWED_EXTENSIONS = {"pdf", "docx", "doc", "pptx", "ppt", "png", "jpg", "jpeg", "html"} +ALLOWED_EXTENSIONS = {"pdf", "docx", "doc", "pptx", "ppt", "png", "jpg", "jpeg", "html", "txt"} MAX_FILE_SIZE_MB = 200 diff --git a/backend/services/indexing_service.py b/backend/services/indexing_service.py index b359108..d489974 100644 --- a/backend/services/indexing_service.py +++ b/backend/services/indexing_service.py @@ -80,57 +80,91 @@ def _run_pipeline(job_id: str) -> None: _update_meta(job_id, status="cancelled", stage="Cancelled") return - _update_meta(job_id, status="parsing", stage="MinerU document parsing...") - mineru_out_dir = job_dir / "mineru_output" - mineru_out_dir.mkdir(parents=True, exist_ok=True) - - result = subprocess.run( - [str(MINERU_PYTHON), str(MINERU_PIPELINE), str(pdf_path)], - cwd=str(MINERU_PIPELINE.parent), - capture_output=True, - text=True, - timeout=600, - ) - - if result.returncode != 0: - raise RuntimeError(f"MinerU failed: {result.stderr[:500]}") - - # Find content_list.json in MinerU output - # MinerU writes output to mineru_mvp/output/{stem}/ - stem = pdf_path.stem - mineru_default_out = MINERU_PIPELINE.parent / "output" / stem - content_list_path = None - - if mineru_default_out.exists(): - matches = list(mineru_default_out.glob("*_content_list.json")) - if matches: - content_list_path = matches[0] - # Copy to our job dir - import shutil - shutil.copytree(str(mineru_default_out), str(mineru_out_dir), dirs_exist_ok=True) - - if not content_list_path: - # Fallback: search job mineru_output dir - matches = list(mineru_out_dir.glob("*_content_list.json")) - if matches: - content_list_path = matches[0] - - if not content_list_path or not content_list_path.exists(): - raise RuntimeError(f"MinerU output content_list.json not found. stdout: {result.stdout[:300]}") - - # ── Stage 2: extracting ─────────────────────────────────────────── - if _cancel_flags.get(job_id): - _update_meta(job_id, status="cancelled", stage="Cancelled") - return - - from pipeline.text_assembler import load_content_list, assemble_pages, count_blocks_by_type + from pipeline.text_assembler import PageText, BlockSpan from pipeline.entity_extractor import create_model, extract_entities from pipeline.kg_builder import build_kg, extractions_to_records - content_list = load_content_list(content_list_path) - pages = assemble_pages(content_list) - total_pages = len(pages) - block_types = count_blocks_by_type(content_list) + ext = pdf_path.suffix.lower().lstrip(".") + + if ext in ("txt",): + # Plain text — skip MinerU, split by paragraphs + _update_meta(job_id, status="parsing", stage="Parsing plain text...") + raw = pdf_path.read_text(encoding="utf-8") + + # Split into ~3000-char pages at double-newline boundaries + PAGE_SIZE = 3000 + raw_pages = [] + pos = 0 + while pos < len(raw): + end = min(pos + PAGE_SIZE, len(raw)) + if end < len(raw): + # Try to break at a double newline + br = raw.rfind("\n\n", pos, end) + if br > pos + PAGE_SIZE // 2: + end = br + 2 + raw_pages.append(raw[pos:end]) + pos = end + + pages = [] + for i, text in enumerate(raw_pages): + pages.append(PageText( + page_idx=i, + text=text.strip(), + block_spans=[BlockSpan( + block_index=0, block_type="text", + page_idx=i, char_start=0, + char_end=len(text.strip()), bbox=[0, 0, 0, 0], + )], + )) + total_pages = len(pages) + block_types = {"text": total_pages} + + # Save full.txt in mineru_output for reference + mineru_out_dir = job_dir / "mineru_output" + mineru_out_dir.mkdir(parents=True, exist_ok=True) + (mineru_out_dir / "full.md").write_text(raw, encoding="utf-8") + else: + _update_meta(job_id, status="parsing", stage="MinerU document parsing...") + mineru_out_dir = job_dir / "mineru_output" + mineru_out_dir.mkdir(parents=True, exist_ok=True) + + result = subprocess.run( + [str(MINERU_PYTHON), str(MINERU_PIPELINE), str(pdf_path)], + cwd=str(MINERU_PIPELINE.parent), + capture_output=True, + text=True, + timeout=600, + ) + + if result.returncode != 0: + raise RuntimeError(f"MinerU failed: {result.stderr[:500]}") + + # Find content_list.json in MinerU output + from pipeline.text_assembler import load_content_list, assemble_pages, count_blocks_by_type + + stem = pdf_path.stem + mineru_default_out = MINERU_PIPELINE.parent / "output" / stem + content_list_path = None + + if mineru_default_out.exists(): + matches = list(mineru_default_out.glob("*_content_list.json")) + if matches: + content_list_path = matches[0] + import shutil + shutil.copytree(str(mineru_default_out), str(mineru_out_dir), dirs_exist_ok=True) + + if not content_list_path: + matches = list(mineru_out_dir.glob("*_content_list.json")) + if matches: + content_list_path = matches[0] + + if not content_list_path or not content_list_path.exists(): + raise RuntimeError(f"MinerU output content_list.json not found. stdout: {result.stdout[:300]}") + + content_list = load_content_list(content_list_path) + pages = assemble_pages(content_list) + total_pages = len(pages) + block_types = count_blocks_by_type(content_list) _update_meta( job_id,