Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 005fe504ad | |||
| 71676dba9e | |||
| a46ceb94de | |||
| 4f203e3b56 | |||
| d2f782e47e |
@@ -19,30 +19,41 @@ DEEPSEEK_BASE_URL = os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com")
|
||||
MODEL_ID = "deepseek-chat"
|
||||
|
||||
PROMPT_DESCRIPTION = (
|
||||
"Extract named entities from the text in order of appearance. "
|
||||
"Entity types: TECHNOLOGY (software, algorithms, models, tools), "
|
||||
"ORGANIZATION (companies, research groups, institutions), "
|
||||
"PERSON (individual people), "
|
||||
"LOCATION (places, geographic entities), "
|
||||
"CONCEPT (technical concepts, methodologies, frameworks)."
|
||||
"从建筑规范文本中按出现顺序提取命名实体。"
|
||||
"实体类型:"
|
||||
"构件 — 建筑构件和部位(防火墙、防火门、楼梯间、电梯井、管道井、变形缝、外墙、屋面、楼板、梁、柱、承重墙、隔墙、门窗、幕墙、雨篷、阳台、走廊、前室、竖井等);"
|
||||
"设备 — 消防设备和系统(消火栓、灭火器、喷淋系统、报警系统、防烟排烟系统、应急照明、疏散指示、防火卷帘、消防电梯、消防水箱、消防水泵、气体灭火系统、火灾自动报警系统、电气火灾监控系统等);"
|
||||
"设施 — 建筑设施和配套(停车场、锅炉房、变配电室、发电机房、储油间、空调机房、通风机房、电梯机房、消防控制室、水泵房等);"
|
||||
"指标 — 技术参数和量化要求(耐火等级、防火间距、疏散宽度、最大面积、高度限值、距离要求、时间要求、容量要求、温度限值、压力要求等);"
|
||||
"场所 — 建筑分类和空间类型(高层建筑、地下建筑、公共建筑、住宅建筑、工业建筑、商业建筑、医疗建筑、教育建筑、娱乐场所、仓库、厂房、中庭、避难层、避难走道、敞开楼梯间等);"
|
||||
"材料 — 建筑材料和产品(不燃材料、难燃材料、可燃材料、防火涂料、防火玻璃、防火密封件、保温材料、装饰材料、钢结构防火保护材料等);"
|
||||
"措施 — 防火措施和策略(防火分隔、防火封堵、自然排烟、机械加压送风、机械排烟、安全疏散、防火保护、消防供电、消防水源、灭火救援等);"
|
||||
"条款 — 规范条文引用(强制性条文、推荐性条文、术语定义、一般规定、基本要求等)。"
|
||||
)
|
||||
|
||||
EXAMPLES = [
|
||||
lx.data.ExampleData(
|
||||
text=(
|
||||
"LangChain is a framework created by Harrison Chase for building "
|
||||
"LLM applications. It integrates with OpenAI models and Pinecone "
|
||||
"vector database for semantic search."
|
||||
),
|
||||
text="公共建筑应设置防火墙。甲级防火门耐火极限为2.0h。消防控制室采用不燃材料。",
|
||||
extractions=[
|
||||
lx.data.Extraction(extraction_class="TECHNOLOGY", extraction_text="LangChain"),
|
||||
lx.data.Extraction(extraction_class="PERSON", extraction_text="Harrison Chase"),
|
||||
lx.data.Extraction(extraction_class="CONCEPT", extraction_text="LLM applications"),
|
||||
lx.data.Extraction(extraction_class="TECHNOLOGY", extraction_text="OpenAI models"),
|
||||
lx.data.Extraction(extraction_class="TECHNOLOGY", extraction_text="Pinecone"),
|
||||
lx.data.Extraction(extraction_class="CONCEPT", extraction_text="semantic search"),
|
||||
lx.data.Extraction(extraction_class="场所", extraction_text="公共建筑"),
|
||||
lx.data.Extraction(extraction_class="构件", extraction_text="防火墙"),
|
||||
lx.data.Extraction(extraction_class="构件", extraction_text="甲级防火门"),
|
||||
lx.data.Extraction(extraction_class="指标", extraction_text="2.0h"),
|
||||
lx.data.Extraction(extraction_class="设施", extraction_text="消防控制室"),
|
||||
lx.data.Extraction(extraction_class="材料", extraction_text="不燃材料"),
|
||||
],
|
||||
)
|
||||
),
|
||||
lx.data.ExampleData(
|
||||
text="住宅建筑应设室内消火栓。高位消防水箱容积为18m3。消防电梯前室设安全出口。",
|
||||
extractions=[
|
||||
lx.data.Extraction(extraction_class="场所", extraction_text="住宅建筑"),
|
||||
lx.data.Extraction(extraction_class="设备", extraction_text="室内消火栓"),
|
||||
lx.data.Extraction(extraction_class="设备", extraction_text="高位消防水箱"),
|
||||
lx.data.Extraction(extraction_class="指标", extraction_text="18m3"),
|
||||
lx.data.Extraction(extraction_class="构件", extraction_text="消防电梯前室"),
|
||||
lx.data.Extraction(extraction_class="构件", extraction_text="安全出口"),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -83,17 +83,14 @@ def make_tools(G: nx.Graph) -> list:
|
||||
def get_entities_by_type(entity_type: str) -> str:
|
||||
"""List all entities of a specific type.
|
||||
Args:
|
||||
entity_type: TECHNOLOGY, CONCEPT, PERSON, ORGANIZATION, or LOCATION.
|
||||
entity_type: 构件, 设备, 设施, 指标, 场所, 材料, 措施, 条款, etc.
|
||||
"""
|
||||
t_upper = entity_type.strip().upper()
|
||||
valid = {"TECHNOLOGY", "CONCEPT", "PERSON", "ORGANIZATION", "LOCATION"}
|
||||
if t_upper not in valid:
|
||||
present = sorted({d.get("type","") for _, d in G.nodes(data=True)})
|
||||
return f"Unknown type '{entity_type}'. Present: {present}"
|
||||
matches = [d for _, d in G.nodes(data=True) if d.get("type","") == t_upper]
|
||||
t = entity_type.strip()
|
||||
matches = [d for _, d in G.nodes(data=True) if d.get("type","") == t]
|
||||
if not matches:
|
||||
return f"No {t_upper} entities found."
|
||||
lines = [f"Found {len(matches)} {t_upper} entities:"]
|
||||
present = sorted({d.get("type","") for _, d in G.nodes(data=True)})
|
||||
return f"No '{entity_type}' entities found. Present types: {present}"
|
||||
lines = [f"Found {len(matches)} {t} entities:"]
|
||||
for m in matches[:30]:
|
||||
lines.append(f" \"{m['name']}\" (page={m.get('page',0)}, id={m['id']})")
|
||||
if len(matches) > 30:
|
||||
@@ -148,15 +145,17 @@ def run_qa(
|
||||
)
|
||||
|
||||
system_prompt = (
|
||||
"You are a helpful assistant with access to a knowledge graph (KG) built from the user's documents.\n"
|
||||
"You are a helpful assistant with access to a knowledge graph (KG) built from building code documents.\n"
|
||||
"The KG contains entities extracted from building fire protection codes (建筑防火规范).\n"
|
||||
"\n"
|
||||
"Entity types include: 构件 (building components), 设备 (fire equipment), 设施 (facilities),\n"
|
||||
"指标 (technical parameters), 场所 (building types/spaces), 材料 (materials),\n"
|
||||
"措施 (fire measures), 条款 (regulation clauses).\n"
|
||||
"\n"
|
||||
"Guidelines:\n"
|
||||
"- If the question is clearly unrelated to the KG (greetings, math, general knowledge, etc.), "
|
||||
"answer directly WITHOUT using any tools.\n"
|
||||
"- If the question might be answered by the KG (topics related to entities in the documents), "
|
||||
"use the tools to search and explore before answering.\n"
|
||||
"- When you DO use the KG, cite the entity names and types you found.\n"
|
||||
"- If the KG has no relevant information, say so honestly and answer from general knowledge if possible.\n"
|
||||
"- Use tools to search and explore the KG before answering.\n"
|
||||
"- Cite the entity names and types you found.\n"
|
||||
"- If the KG has no relevant information, say so honestly.\n"
|
||||
"\n"
|
||||
"Available tools: search entities by name, get neighbors, list entities by type, get graph overview."
|
||||
)
|
||||
|
||||
@@ -110,6 +110,7 @@ async def list_formats():
|
||||
{"ext": "jpg", "description": "JPEG 图片(单页)", "max_size_mb": 200, "max_pages": 1, "requires_ocr": True},
|
||||
{"ext": "jpeg", "description": "JPEG 图片(单页)", "max_size_mb": 200, "max_pages": 1, "requires_ocr": True},
|
||||
{"ext": "html", "description": "HTML 文件", "max_size_mb": 200, "max_pages": 600, "requires_ocr": False},
|
||||
{"ext": "txt", "description": "纯文本文件(UTF-8 编码)", "max_size_mb": 200, "max_pages": 0, "requires_ocr": False},
|
||||
],
|
||||
"ocr_languages": [
|
||||
{"code": "ch", "name": "中文(默认)"},
|
||||
|
||||
@@ -7,7 +7,7 @@ from pathlib import Path
|
||||
|
||||
from storage import file_store as fs
|
||||
|
||||
ALLOWED_EXTENSIONS = {"pdf", "docx", "doc", "pptx", "ppt", "png", "jpg", "jpeg", "html"}
|
||||
ALLOWED_EXTENSIONS = {"pdf", "docx", "doc", "pptx", "ppt", "png", "jpg", "jpeg", "html", "txt"}
|
||||
MAX_FILE_SIZE_MB = 200
|
||||
|
||||
|
||||
|
||||
@@ -80,57 +80,91 @@ def _run_pipeline(job_id: str) -> None:
|
||||
_update_meta(job_id, status="cancelled", stage="Cancelled")
|
||||
return
|
||||
|
||||
_update_meta(job_id, status="parsing", stage="MinerU document parsing...")
|
||||
mineru_out_dir = job_dir / "mineru_output"
|
||||
mineru_out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
result = subprocess.run(
|
||||
[str(MINERU_PYTHON), str(MINERU_PIPELINE), str(pdf_path)],
|
||||
cwd=str(MINERU_PIPELINE.parent),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"MinerU failed: {result.stderr[:500]}")
|
||||
|
||||
# Find content_list.json in MinerU output
|
||||
# MinerU writes output to mineru_mvp/output/{stem}/
|
||||
stem = pdf_path.stem
|
||||
mineru_default_out = MINERU_PIPELINE.parent / "output" / stem
|
||||
content_list_path = None
|
||||
|
||||
if mineru_default_out.exists():
|
||||
matches = list(mineru_default_out.glob("*_content_list.json"))
|
||||
if matches:
|
||||
content_list_path = matches[0]
|
||||
# Copy to our job dir
|
||||
import shutil
|
||||
shutil.copytree(str(mineru_default_out), str(mineru_out_dir), dirs_exist_ok=True)
|
||||
|
||||
if not content_list_path:
|
||||
# Fallback: search job mineru_output dir
|
||||
matches = list(mineru_out_dir.glob("*_content_list.json"))
|
||||
if matches:
|
||||
content_list_path = matches[0]
|
||||
|
||||
if not content_list_path or not content_list_path.exists():
|
||||
raise RuntimeError(f"MinerU output content_list.json not found. stdout: {result.stdout[:300]}")
|
||||
|
||||
# ── Stage 2: extracting ───────────────────────────────────────────
|
||||
if _cancel_flags.get(job_id):
|
||||
_update_meta(job_id, status="cancelled", stage="Cancelled")
|
||||
return
|
||||
|
||||
from pipeline.text_assembler import load_content_list, assemble_pages, count_blocks_by_type
|
||||
from pipeline.text_assembler import PageText, BlockSpan
|
||||
from pipeline.entity_extractor import create_model, extract_entities
|
||||
from pipeline.kg_builder import build_kg, extractions_to_records
|
||||
|
||||
content_list = load_content_list(content_list_path)
|
||||
pages = assemble_pages(content_list)
|
||||
total_pages = len(pages)
|
||||
block_types = count_blocks_by_type(content_list)
|
||||
ext = pdf_path.suffix.lower().lstrip(".")
|
||||
|
||||
if ext in ("txt",):
|
||||
# Plain text — skip MinerU, split by paragraphs
|
||||
_update_meta(job_id, status="parsing", stage="Parsing plain text...")
|
||||
raw = pdf_path.read_text(encoding="utf-8")
|
||||
|
||||
# Split into ~3000-char pages at double-newline boundaries
|
||||
PAGE_SIZE = 3000
|
||||
raw_pages = []
|
||||
pos = 0
|
||||
while pos < len(raw):
|
||||
end = min(pos + PAGE_SIZE, len(raw))
|
||||
if end < len(raw):
|
||||
# Try to break at a double newline
|
||||
br = raw.rfind("\n\n", pos, end)
|
||||
if br > pos + PAGE_SIZE // 2:
|
||||
end = br + 2
|
||||
raw_pages.append(raw[pos:end])
|
||||
pos = end
|
||||
|
||||
pages = []
|
||||
for i, text in enumerate(raw_pages):
|
||||
pages.append(PageText(
|
||||
page_idx=i,
|
||||
text=text.strip(),
|
||||
block_spans=[BlockSpan(
|
||||
block_index=0, block_type="text",
|
||||
page_idx=i, char_start=0,
|
||||
char_end=len(text.strip()), bbox=[0, 0, 0, 0],
|
||||
)],
|
||||
))
|
||||
total_pages = len(pages)
|
||||
block_types = {"text": total_pages}
|
||||
|
||||
# Save full.txt in mineru_output for reference
|
||||
mineru_out_dir = job_dir / "mineru_output"
|
||||
mineru_out_dir.mkdir(parents=True, exist_ok=True)
|
||||
(mineru_out_dir / "full.md").write_text(raw, encoding="utf-8")
|
||||
else:
|
||||
_update_meta(job_id, status="parsing", stage="MinerU document parsing...")
|
||||
mineru_out_dir = job_dir / "mineru_output"
|
||||
mineru_out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
result = subprocess.run(
|
||||
[str(MINERU_PYTHON), str(MINERU_PIPELINE), str(pdf_path)],
|
||||
cwd=str(MINERU_PIPELINE.parent),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"MinerU failed: {result.stderr[:500]}")
|
||||
|
||||
# Find content_list.json in MinerU output
|
||||
from pipeline.text_assembler import load_content_list, assemble_pages, count_blocks_by_type
|
||||
|
||||
stem = pdf_path.stem
|
||||
mineru_default_out = MINERU_PIPELINE.parent / "output" / stem
|
||||
content_list_path = None
|
||||
|
||||
if mineru_default_out.exists():
|
||||
matches = list(mineru_default_out.glob("*_content_list.json"))
|
||||
if matches:
|
||||
content_list_path = matches[0]
|
||||
import shutil
|
||||
shutil.copytree(str(mineru_default_out), str(mineru_out_dir), dirs_exist_ok=True)
|
||||
|
||||
if not content_list_path:
|
||||
matches = list(mineru_out_dir.glob("*_content_list.json"))
|
||||
if matches:
|
||||
content_list_path = matches[0]
|
||||
|
||||
if not content_list_path or not content_list_path.exists():
|
||||
raise RuntimeError(f"MinerU output content_list.json not found. stdout: {result.stdout[:300]}")
|
||||
|
||||
content_list = load_content_list(content_list_path)
|
||||
pages = assemble_pages(content_list)
|
||||
total_pages = len(pages)
|
||||
block_types = count_blocks_by_type(content_list)
|
||||
|
||||
_update_meta(
|
||||
job_id,
|
||||
@@ -184,7 +218,7 @@ def _run_pipeline(job_id: str) -> None:
|
||||
|
||||
elapsed = round(time.time() - start_time, 1)
|
||||
stats = {
|
||||
"blocks": len(content_list),
|
||||
"blocks": total_pages if ext in ("txt",) else len(content_list),
|
||||
"block_types": block_types,
|
||||
"pages": total_pages,
|
||||
"raw_extractions": len(records),
|
||||
|
||||
+6
-2
@@ -2,6 +2,8 @@ server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
client_max_body_size 200m;
|
||||
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
@@ -10,9 +12,11 @@ server {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
# Proxy API requests to backend
|
||||
# Proxy API requests to backend — use variable to defer DNS resolution
|
||||
location /api/ {
|
||||
proxy_pass http://backend:8000;
|
||||
resolver 10.43.0.10 valid=5s;
|
||||
set $backend_upstream backend:8000;
|
||||
proxy_pass http://$backend_upstream;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
|
||||
@@ -32,11 +32,13 @@ async function request<T>(
|
||||
if (parts.length) url += '?' + parts.join('&');
|
||||
}
|
||||
|
||||
// Build headers with optional Bearer token
|
||||
// Build headers with optional Bearer token — refresh if near expiry
|
||||
const headers: Record<string, string> = {};
|
||||
try {
|
||||
const { getToken } = await import('./auth');
|
||||
const token = getToken();
|
||||
const mod = await import('./auth');
|
||||
// Ensure token is valid for at least 30s before sending
|
||||
await mod.default.updateToken(30);
|
||||
const token = mod.getToken();
|
||||
if (token) headers['Authorization'] = `Bearer ${token}`;
|
||||
} catch { /* auth not initialized */ }
|
||||
if (!options.formData && options.body !== undefined) {
|
||||
|
||||
@@ -5,7 +5,7 @@ import { ZoomIn, ZoomOut, Maximize2, Search, Download, Image, X, MessageSquare,
|
||||
import { useAppState, type KGNode } from '../../store';
|
||||
import { TYPE_COLORS } from '../../mock-data';
|
||||
|
||||
const ENTITY_TYPES = ['TECHNOLOGY', 'CONCEPT', 'PERSON', 'ORGANIZATION', 'LOCATION'] as const;
|
||||
const ENTITY_TYPES = ['构件', '设备', '设施', '指标', '场所', '材料', '措施', '条款', 'TECHNOLOGY', 'CONCEPT', 'PERSON', 'ORGANIZATION', 'LOCATION'] as const;
|
||||
const CONFIDENCE_LEVELS = ['match_exact', 'match_greater', 'match_lesser', 'match_fuzzy'] as const;
|
||||
|
||||
export function KGExplorer() {
|
||||
|
||||
@@ -6,7 +6,7 @@ import { useAppState, mapApiNode, mapApiEdge, type KGNode } from '../../store';
|
||||
import { api, ApiError } from '../../api';
|
||||
import { TYPE_COLORS } from '../../mock-data';
|
||||
|
||||
const ENTITY_TYPES_OPTIONS = ['全部类型', 'TECHNOLOGY', 'CONCEPT', 'PERSON', 'ORGANIZATION', 'LOCATION'];
|
||||
const ENTITY_TYPES_OPTIONS = ['全部类型', '构件', '设备', '设施', '指标', '场所', '材料', '措施', '条款', 'TECHNOLOGY', 'CONCEPT', 'PERSON', 'ORGANIZATION', 'LOCATION'];
|
||||
|
||||
export function SearchPage() {
|
||||
const { nodes, edges, getNeighbors } = useAppState();
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
export interface KGNode {
|
||||
id: string;
|
||||
name: string;
|
||||
type: 'TECHNOLOGY' | 'CONCEPT' | 'PERSON' | 'ORGANIZATION' | 'LOCATION';
|
||||
type: '构件' | '设备' | '设施' | '指标' | '场所' | '材料' | '措施' | '条款' | 'TECHNOLOGY' | 'CONCEPT' | 'PERSON' | 'ORGANIZATION' | 'LOCATION';
|
||||
page: number;
|
||||
confidence: 'match_exact' | 'match_greater' | 'match_lesser' | 'match_fuzzy';
|
||||
degree: number;
|
||||
@@ -66,6 +66,14 @@ export interface HistoryItem {
|
||||
|
||||
// Entity type colors
|
||||
export const TYPE_COLORS: Record<string, string> = {
|
||||
'构件': '#58a6ff',
|
||||
'设备': '#ff7b72',
|
||||
'设施': '#d29922',
|
||||
'指标': '#3fb950',
|
||||
'场所': '#bc8cff',
|
||||
'材料': '#ffa657',
|
||||
'措施': '#f0f6fc',
|
||||
'条款': '#8b949e',
|
||||
TECHNOLOGY: '#58a6ff',
|
||||
CONCEPT: '#bc8cff',
|
||||
PERSON: '#3fb950',
|
||||
|
||||
@@ -6,7 +6,7 @@ import { api, type ApiDoc, type ApiKGNode, type ApiKGEdge, ApiError } from './ap
|
||||
export interface KGNode {
|
||||
id: string;
|
||||
name: string;
|
||||
type: 'TECHNOLOGY' | 'CONCEPT' | 'PERSON' | 'ORGANIZATION' | 'LOCATION';
|
||||
type: '构件' | '设备' | '设施' | '指标' | '场所' | '材料' | '措施' | '条款' | 'TECHNOLOGY' | 'CONCEPT' | 'PERSON' | 'ORGANIZATION' | 'LOCATION';
|
||||
page: number;
|
||||
confidence: 'match_exact' | 'match_greater' | 'match_lesser' | 'match_fuzzy';
|
||||
degree: number;
|
||||
@@ -169,9 +169,9 @@ const DEFAULT_STATS: StatsData = { kg_nodes: 0, kg_edges: 0, documents: 0, queri
|
||||
|
||||
const SUGGESTED_PROMPTS = [
|
||||
'给我一个知识图谱的概览',
|
||||
'列出所有 TECHNOLOGY 实体',
|
||||
'GraphRAG 与知识图谱有什么关系?',
|
||||
'什么是检索增强生成?',
|
||||
'列出所有 构件 实体',
|
||||
'列出所有 设备 实体',
|
||||
'文档中有哪些防火措施?',
|
||||
];
|
||||
|
||||
// ─── Provider ─────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -121,6 +121,8 @@ server {
|
||||
ssl_certificate_key /etc/letsencrypt/live/plfai.cn/privkey.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
|
||||
client_max_body_size 200m;
|
||||
|
||||
location / {
|
||||
proxy_pass http://${backend_ip}:8000;
|
||||
proxy_set_header Host \$host;
|
||||
@@ -146,6 +148,8 @@ server {
|
||||
ssl_certificate_key /etc/letsencrypt/live/plfai.cn/privkey.pem;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
|
||||
client_max_body_size 200m;
|
||||
|
||||
location /api/ {
|
||||
proxy_pass http://${backend_ip}:8000;
|
||||
proxy_set_header Host \$host;
|
||||
|
||||
Reference in New Issue
Block a user