5 Commits

Author SHA1 Message Date
admin 005fe504ad fix: 修复 txt 索引 content_list 变量未定义 + 简化示例对齐
GraphRAG CI/CD / build-and-deploy (push) Successful in 5m12s
- txt 流程中 content_list 不存在导致 stats 阶段崩溃
- Few-shot 示例简化,使用短句精确匹配避免对齐失败
2026-06-18 17:10:05 +08:00
admin 71676dba9e feat: 新增 .txt 纯文本文件支持
GraphRAG CI/CD / build-and-deploy (push) Successful in 5m4s
- 添加 txt 到允许上传格式,跳过 MinerU OCR
- 直接读取 UTF-8 文本,按 3000 字符/页分页后送入 LangExtract
- 适合有中文文本的规范文件,避免扫描件 OCR 乱码问题
2026-06-18 17:00:40 +08:00
admin a46ceb94de feat: 重构图谱抽取 — 支持建筑规范文档的 8 种实体类型
GraphRAG CI/CD / build-and-deploy (push) Successful in 6m15s
- 实体类型改为:构件/设备/设施/指标/场所/材料/措施/条款
- 中文 prompt + 建筑规范领域 Few-shot 示例
- 前端图谱颜色、筛选项同步更新
- QA agent 系统提示词适配建筑规范领域
2026-06-18 16:30:58 +08:00
admin 4f203e3b56 fix: nginx 添加 client_max_body_size 200m 支持大文件上传
GraphRAG CI/CD / build-and-deploy (push) Successful in 2m29s
2026-06-18 16:08:23 +08:00
admin d2f782e47e fix: API 请求前用 keycloak.updateToken(30) 刷新即将过期的 token
GraphRAG CI/CD / build-and-deploy (push) Successful in 3m7s
getToken() 只返回缓存的 token,可能已过期(默认 5 分钟生命周期)。
updateToken(30) 确保发送前 token 至少还有 30 秒有效时长。
2026-06-18 16:02:07 +08:00
12 changed files with 159 additions and 96 deletions
+29 -18
View File
@@ -19,30 +19,41 @@ DEEPSEEK_BASE_URL = os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com")
MODEL_ID = "deepseek-chat"
PROMPT_DESCRIPTION = (
"Extract named entities from the text in order of appearance. "
"Entity types: TECHNOLOGY (software, algorithms, models, tools), "
"ORGANIZATION (companies, research groups, institutions), "
"PERSON (individual people), "
"LOCATION (places, geographic entities), "
"CONCEPT (technical concepts, methodologies, frameworks)."
"从建筑规范文本中按出现顺序提取命名实体。"
"实体类型:"
"构件 — 建筑构件和部位(防火墙、防火门、楼梯间、电梯井、管道井、变形缝、外墙、屋面、楼板、梁、柱、承重墙、隔墙、门窗、幕墙、雨篷、阳台、走廊、前室、竖井等);"
"设备 — 消防设备和系统(消火栓、灭火器、喷淋系统、报警系统、防烟排烟系统、应急照明、疏散指示、防火卷帘、消防电梯、消防水箱、消防水泵、气体灭火系统、火灾自动报警系统、电气火灾监控系统等);"
"设施 — 建筑设施和配套(停车场、锅炉房、变配电室、发电机房、储油间、空调机房、通风机房、电梯机房、消防控制室、水泵房等);"
"指标 — 技术参数和量化要求(耐火等级、防火间距、疏散宽度、最大面积、高度限值、距离要求、时间要求、容量要求、温度限值、压力要求等);"
"场所 — 建筑分类和空间类型(高层建筑、地下建筑、公共建筑、住宅建筑、工业建筑、商业建筑、医疗建筑、教育建筑、娱乐场所、仓库、厂房、中庭、避难层、避难走道、敞开楼梯间等);"
"材料 — 建筑材料和产品(不燃材料、难燃材料、可燃材料、防火涂料、防火玻璃、防火密封件、保温材料、装饰材料、钢结构防火保护材料等);"
"措施 — 防火措施和策略(防火分隔、防火封堵、自然排烟、机械加压送风、机械排烟、安全疏散、防火保护、消防供电、消防水源、灭火救援等);"
"条款 — 规范条文引用(强制性条文、推荐性条文、术语定义、一般规定、基本要求等)。"
)
EXAMPLES = [
lx.data.ExampleData(
text=(
"LangChain is a framework created by Harrison Chase for building "
"LLM applications. It integrates with OpenAI models and Pinecone "
"vector database for semantic search."
),
text="公共建筑应设置防火墙。甲级防火门耐火极限为2.0h。消防控制室采用不燃材料。",
extractions=[
lx.data.Extraction(extraction_class="TECHNOLOGY", extraction_text="LangChain"),
lx.data.Extraction(extraction_class="PERSON", extraction_text="Harrison Chase"),
lx.data.Extraction(extraction_class="CONCEPT", extraction_text="LLM applications"),
lx.data.Extraction(extraction_class="TECHNOLOGY", extraction_text="OpenAI models"),
lx.data.Extraction(extraction_class="TECHNOLOGY", extraction_text="Pinecone"),
lx.data.Extraction(extraction_class="CONCEPT", extraction_text="semantic search"),
lx.data.Extraction(extraction_class="场所", extraction_text="公共建筑"),
lx.data.Extraction(extraction_class="构件", extraction_text="防火墙"),
lx.data.Extraction(extraction_class="构件", extraction_text="甲级防火门"),
lx.data.Extraction(extraction_class="指标", extraction_text="2.0h"),
lx.data.Extraction(extraction_class="设施", extraction_text="消防控制室"),
lx.data.Extraction(extraction_class="材料", extraction_text="不燃材料"),
],
)
),
lx.data.ExampleData(
text="住宅建筑应设室内消火栓。高位消防水箱容积为18m3。消防电梯前室设安全出口。",
extractions=[
lx.data.Extraction(extraction_class="场所", extraction_text="住宅建筑"),
lx.data.Extraction(extraction_class="设备", extraction_text="室内消火栓"),
lx.data.Extraction(extraction_class="设备", extraction_text="高位消防水箱"),
lx.data.Extraction(extraction_class="指标", extraction_text="18m3"),
lx.data.Extraction(extraction_class="构件", extraction_text="消防电梯前室"),
lx.data.Extraction(extraction_class="构件", extraction_text="安全出口"),
],
),
]
+15 -16
View File
@@ -83,17 +83,14 @@ def make_tools(G: nx.Graph) -> list:
def get_entities_by_type(entity_type: str) -> str:
"""List all entities of a specific type.
Args:
entity_type: TECHNOLOGY, CONCEPT, PERSON, ORGANIZATION, or LOCATION.
entity_type: 构件, 设备, 设施, 指标, 场所, 材料, 措施, 条款, etc.
"""
t_upper = entity_type.strip().upper()
valid = {"TECHNOLOGY", "CONCEPT", "PERSON", "ORGANIZATION", "LOCATION"}
if t_upper not in valid:
present = sorted({d.get("type","") for _, d in G.nodes(data=True)})
return f"Unknown type '{entity_type}'. Present: {present}"
matches = [d for _, d in G.nodes(data=True) if d.get("type","") == t_upper]
t = entity_type.strip()
matches = [d for _, d in G.nodes(data=True) if d.get("type","") == t]
if not matches:
return f"No {t_upper} entities found."
lines = [f"Found {len(matches)} {t_upper} entities:"]
present = sorted({d.get("type","") for _, d in G.nodes(data=True)})
return f"No '{entity_type}' entities found. Present types: {present}"
lines = [f"Found {len(matches)} {t} entities:"]
for m in matches[:30]:
lines.append(f" \"{m['name']}\" (page={m.get('page',0)}, id={m['id']})")
if len(matches) > 30:
@@ -148,15 +145,17 @@ def run_qa(
)
system_prompt = (
"You are a helpful assistant with access to a knowledge graph (KG) built from the user's documents.\n"
"You are a helpful assistant with access to a knowledge graph (KG) built from building code documents.\n"
"The KG contains entities extracted from building fire protection codes (建筑防火规范).\n"
"\n"
"Entity types include: 构件 (building components), 设备 (fire equipment), 设施 (facilities),\n"
"指标 (technical parameters), 场所 (building types/spaces), 材料 (materials),\n"
"措施 (fire measures), 条款 (regulation clauses).\n"
"\n"
"Guidelines:\n"
"- If the question is clearly unrelated to the KG (greetings, math, general knowledge, etc.), "
"answer directly WITHOUT using any tools.\n"
"- If the question might be answered by the KG (topics related to entities in the documents), "
"use the tools to search and explore before answering.\n"
"- When you DO use the KG, cite the entity names and types you found.\n"
"- If the KG has no relevant information, say so honestly and answer from general knowledge if possible.\n"
"- Use tools to search and explore the KG before answering.\n"
"- Cite the entity names and types you found.\n"
"- If the KG has no relevant information, say so honestly.\n"
"\n"
"Available tools: search entities by name, get neighbors, list entities by type, get graph overview."
)
+1
View File
@@ -110,6 +110,7 @@ async def list_formats():
{"ext": "jpg", "description": "JPEG 图片(单页)", "max_size_mb": 200, "max_pages": 1, "requires_ocr": True},
{"ext": "jpeg", "description": "JPEG 图片(单页)", "max_size_mb": 200, "max_pages": 1, "requires_ocr": True},
{"ext": "html", "description": "HTML 文件", "max_size_mb": 200, "max_pages": 600, "requires_ocr": False},
{"ext": "txt", "description": "纯文本文件(UTF-8 编码)", "max_size_mb": 200, "max_pages": 0, "requires_ocr": False},
],
"ocr_languages": [
{"code": "ch", "name": "中文(默认)"},
+1 -1
View File
@@ -7,7 +7,7 @@ from pathlib import Path
from storage import file_store as fs
ALLOWED_EXTENSIONS = {"pdf", "docx", "doc", "pptx", "ppt", "png", "jpg", "jpeg", "html"}
ALLOWED_EXTENSIONS = {"pdf", "docx", "doc", "pptx", "ppt", "png", "jpg", "jpeg", "html", "txt"}
MAX_FILE_SIZE_MB = 200
+83 -49
View File
@@ -80,57 +80,91 @@ def _run_pipeline(job_id: str) -> None:
_update_meta(job_id, status="cancelled", stage="Cancelled")
return
_update_meta(job_id, status="parsing", stage="MinerU document parsing...")
mineru_out_dir = job_dir / "mineru_output"
mineru_out_dir.mkdir(parents=True, exist_ok=True)
result = subprocess.run(
[str(MINERU_PYTHON), str(MINERU_PIPELINE), str(pdf_path)],
cwd=str(MINERU_PIPELINE.parent),
capture_output=True,
text=True,
timeout=600,
)
if result.returncode != 0:
raise RuntimeError(f"MinerU failed: {result.stderr[:500]}")
# Find content_list.json in MinerU output
# MinerU writes output to mineru_mvp/output/{stem}/
stem = pdf_path.stem
mineru_default_out = MINERU_PIPELINE.parent / "output" / stem
content_list_path = None
if mineru_default_out.exists():
matches = list(mineru_default_out.glob("*_content_list.json"))
if matches:
content_list_path = matches[0]
# Copy to our job dir
import shutil
shutil.copytree(str(mineru_default_out), str(mineru_out_dir), dirs_exist_ok=True)
if not content_list_path:
# Fallback: search job mineru_output dir
matches = list(mineru_out_dir.glob("*_content_list.json"))
if matches:
content_list_path = matches[0]
if not content_list_path or not content_list_path.exists():
raise RuntimeError(f"MinerU output content_list.json not found. stdout: {result.stdout[:300]}")
# ── Stage 2: extracting ───────────────────────────────────────────
if _cancel_flags.get(job_id):
_update_meta(job_id, status="cancelled", stage="Cancelled")
return
from pipeline.text_assembler import load_content_list, assemble_pages, count_blocks_by_type
from pipeline.text_assembler import PageText, BlockSpan
from pipeline.entity_extractor import create_model, extract_entities
from pipeline.kg_builder import build_kg, extractions_to_records
content_list = load_content_list(content_list_path)
pages = assemble_pages(content_list)
total_pages = len(pages)
block_types = count_blocks_by_type(content_list)
ext = pdf_path.suffix.lower().lstrip(".")
if ext in ("txt",):
# Plain text — skip MinerU, split by paragraphs
_update_meta(job_id, status="parsing", stage="Parsing plain text...")
raw = pdf_path.read_text(encoding="utf-8")
# Split into ~3000-char pages at double-newline boundaries
PAGE_SIZE = 3000
raw_pages = []
pos = 0
while pos < len(raw):
end = min(pos + PAGE_SIZE, len(raw))
if end < len(raw):
# Try to break at a double newline
br = raw.rfind("\n\n", pos, end)
if br > pos + PAGE_SIZE // 2:
end = br + 2
raw_pages.append(raw[pos:end])
pos = end
pages = []
for i, text in enumerate(raw_pages):
pages.append(PageText(
page_idx=i,
text=text.strip(),
block_spans=[BlockSpan(
block_index=0, block_type="text",
page_idx=i, char_start=0,
char_end=len(text.strip()), bbox=[0, 0, 0, 0],
)],
))
total_pages = len(pages)
block_types = {"text": total_pages}
# Save full.txt in mineru_output for reference
mineru_out_dir = job_dir / "mineru_output"
mineru_out_dir.mkdir(parents=True, exist_ok=True)
(mineru_out_dir / "full.md").write_text(raw, encoding="utf-8")
else:
_update_meta(job_id, status="parsing", stage="MinerU document parsing...")
mineru_out_dir = job_dir / "mineru_output"
mineru_out_dir.mkdir(parents=True, exist_ok=True)
result = subprocess.run(
[str(MINERU_PYTHON), str(MINERU_PIPELINE), str(pdf_path)],
cwd=str(MINERU_PIPELINE.parent),
capture_output=True,
text=True,
timeout=600,
)
if result.returncode != 0:
raise RuntimeError(f"MinerU failed: {result.stderr[:500]}")
# Find content_list.json in MinerU output
from pipeline.text_assembler import load_content_list, assemble_pages, count_blocks_by_type
stem = pdf_path.stem
mineru_default_out = MINERU_PIPELINE.parent / "output" / stem
content_list_path = None
if mineru_default_out.exists():
matches = list(mineru_default_out.glob("*_content_list.json"))
if matches:
content_list_path = matches[0]
import shutil
shutil.copytree(str(mineru_default_out), str(mineru_out_dir), dirs_exist_ok=True)
if not content_list_path:
matches = list(mineru_out_dir.glob("*_content_list.json"))
if matches:
content_list_path = matches[0]
if not content_list_path or not content_list_path.exists():
raise RuntimeError(f"MinerU output content_list.json not found. stdout: {result.stdout[:300]}")
content_list = load_content_list(content_list_path)
pages = assemble_pages(content_list)
total_pages = len(pages)
block_types = count_blocks_by_type(content_list)
_update_meta(
job_id,
@@ -184,7 +218,7 @@ def _run_pipeline(job_id: str) -> None:
elapsed = round(time.time() - start_time, 1)
stats = {
"blocks": len(content_list),
"blocks": total_pages if ext in ("txt",) else len(content_list),
"block_types": block_types,
"pages": total_pages,
"raw_extractions": len(records),
+6 -2
View File
@@ -2,6 +2,8 @@ server {
listen 80;
server_name _;
client_max_body_size 200m;
root /usr/share/nginx/html;
index index.html;
@@ -10,9 +12,11 @@ server {
try_files $uri $uri/ /index.html;
}
# Proxy API requests to backend
# Proxy API requests to backend — use variable to defer DNS resolution
location /api/ {
proxy_pass http://backend:8000;
resolver 10.43.0.10 valid=5s;
set $backend_upstream backend:8000;
proxy_pass http://$backend_upstream;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+5 -3
View File
@@ -32,11 +32,13 @@ async function request<T>(
if (parts.length) url += '?' + parts.join('&');
}
// Build headers with optional Bearer token
// Build headers with optional Bearer token — refresh if near expiry
const headers: Record<string, string> = {};
try {
const { getToken } = await import('./auth');
const token = getToken();
const mod = await import('./auth');
// Ensure token is valid for at least 30s before sending
await mod.default.updateToken(30);
const token = mod.getToken();
if (token) headers['Authorization'] = `Bearer ${token}`;
} catch { /* auth not initialized */ }
if (!options.formData && options.body !== undefined) {
@@ -5,7 +5,7 @@ import { ZoomIn, ZoomOut, Maximize2, Search, Download, Image, X, MessageSquare,
import { useAppState, type KGNode } from '../../store';
import { TYPE_COLORS } from '../../mock-data';
const ENTITY_TYPES = ['TECHNOLOGY', 'CONCEPT', 'PERSON', 'ORGANIZATION', 'LOCATION'] as const;
const ENTITY_TYPES = ['构件', '设备', '设施', '指标', '场所', '材料', '措施', '条款', 'TECHNOLOGY', 'CONCEPT', 'PERSON', 'ORGANIZATION', 'LOCATION'] as const;
const CONFIDENCE_LEVELS = ['match_exact', 'match_greater', 'match_lesser', 'match_fuzzy'] as const;
export function KGExplorer() {
@@ -6,7 +6,7 @@ import { useAppState, mapApiNode, mapApiEdge, type KGNode } from '../../store';
import { api, ApiError } from '../../api';
import { TYPE_COLORS } from '../../mock-data';
const ENTITY_TYPES_OPTIONS = ['全部类型', 'TECHNOLOGY', 'CONCEPT', 'PERSON', 'ORGANIZATION', 'LOCATION'];
const ENTITY_TYPES_OPTIONS = ['全部类型', '构件', '设备', '设施', '指标', '场所', '材料', '措施', '条款', 'TECHNOLOGY', 'CONCEPT', 'PERSON', 'ORGANIZATION', 'LOCATION'];
export function SearchPage() {
const { nodes, edges, getNeighbors } = useAppState();
+9 -1
View File
@@ -3,7 +3,7 @@
export interface KGNode {
id: string;
name: string;
type: 'TECHNOLOGY' | 'CONCEPT' | 'PERSON' | 'ORGANIZATION' | 'LOCATION';
type: '构件' | '设备' | '设施' | '指标' | '场所' | '材料' | '措施' | '条款' | 'TECHNOLOGY' | 'CONCEPT' | 'PERSON' | 'ORGANIZATION' | 'LOCATION';
page: number;
confidence: 'match_exact' | 'match_greater' | 'match_lesser' | 'match_fuzzy';
degree: number;
@@ -66,6 +66,14 @@ export interface HistoryItem {
// Entity type colors
export const TYPE_COLORS: Record<string, string> = {
'构件': '#58a6ff',
'设备': '#ff7b72',
'设施': '#d29922',
'指标': '#3fb950',
'场所': '#bc8cff',
'材料': '#ffa657',
'措施': '#f0f6fc',
'条款': '#8b949e',
TECHNOLOGY: '#58a6ff',
CONCEPT: '#bc8cff',
PERSON: '#3fb950',
+4 -4
View File
@@ -6,7 +6,7 @@ import { api, type ApiDoc, type ApiKGNode, type ApiKGEdge, ApiError } from './ap
export interface KGNode {
id: string;
name: string;
type: 'TECHNOLOGY' | 'CONCEPT' | 'PERSON' | 'ORGANIZATION' | 'LOCATION';
type: '构件' | '设备' | '设施' | '指标' | '场所' | '材料' | '措施' | '条款' | 'TECHNOLOGY' | 'CONCEPT' | 'PERSON' | 'ORGANIZATION' | 'LOCATION';
page: number;
confidence: 'match_exact' | 'match_greater' | 'match_lesser' | 'match_fuzzy';
degree: number;
@@ -169,9 +169,9 @@ const DEFAULT_STATS: StatsData = { kg_nodes: 0, kg_edges: 0, documents: 0, queri
const SUGGESTED_PROMPTS = [
'给我一个知识图谱的概览',
'列出所有 TECHNOLOGY 实体',
'GraphRAG 与知识图谱有什么关系?',
'什么是检索增强生成',
'列出所有 构件 实体',
'列出所有 设备 实体',
'文档中有哪些防火措施',
];
// ─── Provider ─────────────────────────────────────────────────────────────────
+4
View File
@@ -121,6 +121,8 @@ server {
ssl_certificate_key /etc/letsencrypt/live/plfai.cn/privkey.pem;
ssl_protocols TLSv1.2 TLSv1.3;
client_max_body_size 200m;
location / {
proxy_pass http://${backend_ip}:8000;
proxy_set_header Host \$host;
@@ -146,6 +148,8 @@ server {
ssl_certificate_key /etc/letsencrypt/live/plfai.cn/privkey.pem;
ssl_protocols TLSv1.2 TLSv1.3;
client_max_body_size 200m;
location /api/ {
proxy_pass http://${backend_ip}:8000;
proxy_set_header Host \$host;