"""Step 5 人工验证:对 Step4 实际落盘的章节内容做审核与修复。 输入(均使用已生成中间产物,不重跑 Step1~Step4): - Step2 分析结果 step2_info.pkl(评分项、废标项) - Step3 大纲与评分/废标映射 step3_outline_report.json - Step4 实际章节 DOCX step4_chapters/ 用法(按顺序运行): 1. uv run python scripts/test_step1.py; 2. uv run python scripts/test_step2.py; 3. uv run python scripts/test_step3.py; 4. uv run python scripts/test_step4.py; 5. uv run python scripts/test_step5.py。 """ import glob import json import logging import os import pickle import re import sys import xml.etree.ElementTree as ET from collections import Counter from _bootstrap import PROJECT_ROOT from models import BidOutline, Chapter, ChapterType OUTPUT_DIR = os.environ.get( "PROPOSA_STEP5_OUTPUT_DIR", os.environ.get("PROPOSA_WORK_DIR", "output/171-上海群众艺术馆"), ) os.environ.setdefault("BID_LLM_CACHE_DIR", os.path.join(OUTPUT_DIR, ".llm_cache")) os.environ.setdefault("BID_LLM_CACHE", "0") STEP2_INFO_FILE = os.environ.get( "PROPOSA_STEP2_INFO_FILE", os.path.join(OUTPUT_DIR, "step2_info.pkl"), ) STEP3_REPORT_FILE = os.environ.get( "PROPOSA_STEP3_REPORT_FILE", os.path.join(OUTPUT_DIR, "step3_outline_report.json"), ) STEP4_CHAPTERS_DIR = os.environ.get( "PROPOSA_STEP4_CHAPTERS_DIR", os.path.join(OUTPUT_DIR, "step4_chapters"), ) STEP5_OUTLINE_FILE = os.environ.get( "PROPOSA_STEP5_REVIEWED_OUTLINE_FILE", os.path.join(OUTPUT_DIR, "step5_outline.pkl"), ) STEP5_CHAPTERS_DIR = os.environ.get( "PROPOSA_STEP5_CHAPTERS_DIR", os.path.join(OUTPUT_DIR, "step5_chapters"), ) logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%H:%M:%S", stream=sys.stderr, ) def _load_step2_info(path): if not os.path.isfile(path): raise FileNotFoundError( f"Step2 分析结果不存在: {path}\n" "请先按顺序运行 uv run python scripts/test_step1.py 和 " "uv run python scripts/test_step2.py。" ) with open(path, "rb") as file: payload = pickle.load(file) analysis = payload.get("analysis") project_data = payload.get("project_data") if analysis is None: raise RuntimeError(f"Step2 文件缺少 analysis: {path}") return analysis, project_data def _load_step3_report(path): if not os.path.isfile(path): raise FileNotFoundError( f"Step3 大纲/映射不存在: {path}\n" "请先按顺序运行 uv run python scripts/test_step1.py 至 " "uv run python scripts/test_step3.py。" ) with open(path, "r", encoding="utf-8") as file: return json.load(file) def _find_manifest(chapters_dir): for candidate in glob.glob( os.path.join(chapters_dir, "chapters", "*", "manifest.json") ): return candidate return "" def _load_chapter_texts(chapters_dir): """读取 Step4 实际落盘的每个章节 DOCX 文本。""" from doc_reader import read_file manifest_path = _find_manifest(chapters_dir) records = [] if manifest_path: with open(manifest_path, "r", encoding="utf-8") as file: manifest = json.load(file) records = manifest.get("chapters", []) else: for path in glob.glob( os.path.join(chapters_dir, "chapters", "*", "*.docx") ): base = os.path.basename(path) chapter_id = base.split("_", 1)[0] records.append({"id": chapter_id, "artifact_path": path}) texts = {} for record in records: chapter_id = str(record.get("id", "")) path = record.get("artifact_path", "") if not chapter_id or not path or not os.path.isfile(path): continue content = read_file(path) if content: texts[chapter_id] = content return texts def _load_step4_records(chapters_dir): """读取 Step4 manifest 中的章节文件记录(id/标题/源路径/状态)。""" manifest_path = _find_manifest(chapters_dir) if not manifest_path: records = [] for path in glob.glob( os.path.join(chapters_dir, "chapters", "*", "*.docx") ): base = os.path.basename(path) records.append({ "id": base.split("_", 1)[0], "title": base.rsplit("_", 1)[0], "artifact_path": path, "status": "complete", }) return records with open(manifest_path, "r", encoding="utf-8") as file: manifest = json.load(file) return [ { "id": str(record.get("id", "")), "title": str(record.get("title", "")), "artifact_path": str(record.get("artifact_path", "")), "status": str(record.get("status", "complete")), } for record in manifest.get("chapters", []) if record.get("artifact_path") ] def _chapter_type_for(final_id: str) -> ChapterType: return ( ChapterType.BUSINESS if final_id in {"index", "1", "2"} else ChapterType.TECHNICAL ) def _build_outline_from_report(report, chapter_texts): mappings = list(report.get("heading_mappings", []) or []) entries = list(report.get("evaluation_index_entries", []) or []) nodes = {} for mapping in mappings: final_id = str(mapping.get("final_id", "")) if not final_id: continue level = 1 if final_id == "index" else 1 + final_id.count(".") nodes[final_id] = Chapter( id=final_id, title=str(mapping.get("final_title", "") or ""), chapter_type=_chapter_type_for(final_id), level=level, template_original_title=str(mapping.get("template_title", "") or ""), template_original_id=str(mapping.get("template_id", "") or ""), template_chapter_id=( str(mapping.get("template_id", "") or "") if level == 1 else "" ), content_generation_mode=str( mapping.get("content_generation_mode", "") or "" ), direct_scoring_bindings=list(mapping.get("scoring_bindings", []) or []), direct_rejection_bindings=list(mapping.get("rejection_bindings", []) or []), structure_locked=bool( mapping.get("scoring_bindings") or mapping.get("rejection_bindings") ), ) for entry in entries: heading_id = str(entry.get("final_heading_id", "") or "") node = nodes.get(heading_id) if node is None: continue if entry.get("entry_type") == "scoring": criterion_id = str(entry.get("criterion_id", "") or "") if criterion_id and criterion_id not in node.related_criteria: node.related_criteria.append(criterion_id) if criterion_id and criterion_id not in node.direct_scoring_criteria: node.direct_scoring_criteria.append(criterion_id) elif entry.get("entry_type") == "rejection": source_id = str(entry.get("source_id", "") or "") if source_id and source_id not in node.related_rejections: node.related_rejections.append(source_id) roots = [] for mapping in mappings: final_id = str(mapping.get("final_id", "")) parent_id = str(mapping.get("parent_final_id", "") or "") node = nodes.get(final_id) if node is None: continue parent = nodes.get(parent_id) if parent_id else None if parent is not None: node.chapter_type = parent.chapter_type if node not in parent.children: parent.children.append(node) else: roots.append(node) for chapter in roots: chapter.generated_content = chapter_texts.get(chapter.id, "") outline = BidOutline( project_name=str(report.get("project", "") or "投标项目"), chapters=roots, ) outline.heading_mappings = list(mappings) outline.evaluation_index_entries = list(entries) return outline def _safe_project_name(name): return ( re.sub(r'[<>:"/\\|?*]+', "_", name or "投标项目").strip(" ._") or "投标项目" ) def _normalize_docx_text(text): return re.sub(r"[\s\u3000]+", "", text or "") _EMPTY_LABEL_RE = re.compile( r"^(项目名称|项目编号|招标编号|招标项目编号|包号|包件号|包件名称|包名|" r"服务内容|服务要求|服务期限)[::]\s*$" ) _PROJECT_LABEL_RE = re.compile( r"^(项目名称|项目编号|招标编号|招标项目编号|包号|包件号|包件名称|包名|" r"服务内容|服务要求|服务期限)[::]\s*(.*)$" ) _TITLE_PREFIX_RE = re.compile( r"^(第[一二三四五六七八九十百\d]+章|" r"[一二三四五六七八九十百]+、|" r"[((][一二三四五六七八九十百\d]+[))]|" r"\d+[.、.]|" r"[((]\d+[))]|" r"\d+\)|[a-zA-Z][.、.)])" ) _PROTECTED_SIGNATURE_RE = re.compile( r"^\s*(投标人授权代表签字|投标人(公章)|投标人\(公章\)|日期|法定代表人|" r"授权代表|签署人)[::((]" ) _PACKAGE_LABELS = {"包号", "包件", "包件号", "包件名称", "包名"} _PROJECT_LABEL_ALIASES = { "招标编号": "项目编号", "招标项目编号": "项目编号", "包件号": "包号", "包件名称": "包号", "包名": "包号", } def _heading_level(paragraph): style = getattr(getattr(paragraph, "style", None), "name", "") or "" match = re.match(r"(?:Heading|标题)\s*([1-4])", style, re.I) return int(match.group(1)) if match else 0 def _clear_paragraph_text(paragraph): for run in list(paragraph.runs): run.text = "" def _ensure_body_indent(paragraph): """给正文段落写入首行缩进 2 字符(Word firstLineChars=200)。""" W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" p_pr = paragraph.find(f"{{{W_NS}}}pPr") if p_pr is None: p_pr = paragraph.makeelement(f"{{{W_NS}}}pPr", {}) paragraph.insert(0, p_pr) ind = p_pr.find(f"{{{W_NS}}}ind") if ind is None: ind = p_pr.makeelement(f"{{{W_NS}}}ind", {}) p_pr.append(ind) ind.set(f"{{{W_NS}}}firstLineChars", "200") ind.set(f"{{{W_NS}}}firstLine", "480") def _apply_body_format(paragraph, W_NS): """给正文段落应用宋体小四、黑色、1.5 倍行距与首行缩进。""" p_pr = paragraph.find(f"{{{W_NS}}}pPr") if p_pr is None: p_pr = ET.SubElement(paragraph, f"{{{W_NS}}}pPr") spacing = p_pr.find(f"{{{W_NS}}}spacing") if spacing is None: spacing = ET.SubElement(p_pr, f"{{{W_NS}}}spacing") spacing.set(f"{{{W_NS}}}line", "360") spacing.set(f"{{{W_NS}}}lineRule", "auto") spacing.set(f"{{{W_NS}}}before", "0") spacing.set(f"{{{W_NS}}}after", "0") ind = p_pr.find(f"{{{W_NS}}}ind") if ind is None: ind = ET.SubElement(p_pr, f"{{{W_NS}}}ind") ind.set(f"{{{W_NS}}}firstLineChars", "200") ind.set(f"{{{W_NS}}}firstLine", "480") for run in paragraph.findall(f"{{{W_NS}}}r"): r_pr = run.find(f"{{{W_NS}}}rPr") if r_pr is None: r_pr = ET.SubElement(run, f"{{{W_NS}}}rPr") r_fonts = r_pr.find(f"{{{W_NS}}}rFonts") if r_fonts is None: r_fonts = ET.SubElement(r_pr, f"{{{W_NS}}}rFonts") r_fonts.set(f"{{{W_NS}}}ascii", "Times New Roman") r_fonts.set(f"{{{W_NS}}}hAnsi", "Times New Roman") r_fonts.set(f"{{{W_NS}}}eastAsia", "宋体") for tag, value in (("sz", "24"), ("szCs", "24")): node = r_pr.find(f"{{{W_NS}}}{tag}") if node is None: node = ET.SubElement(r_pr, f"{{{W_NS}}}{tag}") node.set(f"{{{W_NS}}}val", value) color = r_pr.find(f"{{{W_NS}}}color") if color is None: color = ET.SubElement(r_pr, f"{{{W_NS}}}color") color.set(f"{{{W_NS}}}val", "000000") def _clean_chapter_docx(doc, has_packages): """直接在 step4 章节 DOCX 上删除包号行和重复正文段落。 has_packages 由 Step2 的 LLM 提取结果决定;只有确认项目无实际包号信息时, 才删除章节中的包号/包件文字。 """ _PACKAGE_LABELS = {"包号", "包件", "包件号", "包件名称", "包名"} seen = set() for paragraph in doc.paragraphs: text = (paragraph.text or "").strip() if not text: continue level = _heading_level(paragraph) if level == 0: if not has_packages and package_re.search(text): _clear_paragraph_text(paragraph) continue key = _normalize_docx_text(text) if len(key) >= 12 and key in seen: _clear_paragraph_text(paragraph) continue if len(key) >= 12: seen.add(key) if has_packages: return for table in doc.tables: for row in table.rows: for cell in row.cells: for paragraph in cell.paragraphs: if package_re.search(paragraph.text or ""): _clear_paragraph_text(paragraph) def _find_heading_paragraph(doc, node): candidates = [ _normalize_docx_text(value) for value in (node.title, getattr(node, "template_original_title", "")) if value ] for paragraph in doc.paragraphs: if _heading_level(paragraph) and _normalize_docx_text(paragraph.text) in candidates: return paragraph return None def _insert_node_supplement(doc, node): """把 Step5 生成的补充正文插入到对应绑定标题之后,不改动原生内容块。""" content = (getattr(node, "supplement_content", "") or "").strip() if not content: content = (getattr(node, "generated_content", "") or "").strip() if not content: return 0 anchor = _find_heading_paragraph(doc, node) if anchor is None: return 0 existing = { _normalize_docx_text(paragraph.text) for paragraph in doc.paragraphs } lines = [] for raw in content.splitlines(): line = raw.strip() key = _normalize_docx_text(line) if key and key not in existing: lines.append(line) existing.add(key) if not lines: return 0 node_level = int(getattr(node, "level", 2) or 2) boundary = None anchor_index = None for index, paragraph in enumerate(doc.paragraphs): if paragraph is anchor: anchor_index = index continue if anchor_index is not None: level = _heading_level(paragraph) if level and level <= node_level: boundary = paragraph break if boundary is not None: for line in reversed(lines): boundary.insert_paragraph_before(line) else: for line in lines: doc.add_paragraph(line) return len(lines) def _has_package_info(analysis): """依据 Step2 LLM 提取结果判断项目是否真的存在包号/包件信息。""" from step5_reviewing.policies import analysis_has_package_info return analysis_has_package_info(analysis) def _safe_copy_and_clean_chapter(source, target, has_packages, remove_keys=None, fills=None): """用标准库重写 document.xml,避免 python-docx/lxml 在大型章节上 0xC0000005。""" import xml.etree.ElementTree as ET import zipfile W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" package_re = re.compile(r"(包号|包件)") document_name = "word/document.xml" remove_keys = set(remove_keys or []) fills = list(fills or []) with zipfile.ZipFile(source, "r") as source_zip: names = set(source_zip.namelist()) document_out = None if document_name in names: root = ET.fromstring(source_zip.read(document_name)) body = None for element in root.iter(): if element.tag == f"{{{W_NS}}}body": body = element break if body is not None: seen = set() seen_filled_labels = set() for paragraph in list(body): if paragraph.tag != f"{{{W_NS}}}p": continue text_nodes = [ node for node in paragraph.iter() if node.tag == f"{{{W_NS}}}t" ] text = "".join(node.text or "" for node in text_nodes) if not text.strip(): continue if _PROTECTED_SIGNATURE_RE.match(text.strip()): continue key = _normalize_docx_text(text) p_pr = paragraph.find(f"{{{W_NS}}}pPr") style_val = "" if p_pr is not None: p_style = p_pr.find(f"{{{W_NS}}}pStyle") if p_style is not None: style_val = p_style.get(f"{{{W_NS}}}val", "") is_title = bool( _TITLE_PREFIX_RE.match(text.strip()) or re.match(r"(?:Heading|标题)\s*[1-4]", style_val, re.I) ) if is_title: seen = set() seen_filled_labels = set() continue if key in remove_keys: for node in text_nodes: node.text = "" continue label_match = _PROJECT_LABEL_RE.match(text.strip()) if label_match: raw_label = label_match.group(1).strip() label_value = label_match.group(2).strip() label = _PROJECT_LABEL_ALIASES.get(raw_label, raw_label) if label_value: seen_filled_labels.add(label) continue if label in seen_filled_labels: for node in text_nodes: node.text = "" continue if raw_label in _PACKAGE_LABELS and not has_packages: for node in text_nodes: node.text = "" continue continue if len(key) >= 12 and key in seen: for node in text_nodes: node.text = "" continue if len(key) >= 12: seen.add(key) for fill in fills: label = str(fill.get("label", "")).strip() value = str(fill.get("value", "")).strip() if not label or not value: continue head = re.split(r"[::]", text.strip(), maxsplit=1)[0].strip() if _normalize_docx_text(head) == _normalize_docx_text(label): for node in text_nodes: node.text = "" if text_nodes: text_nodes[0].text = f"{label}:{value}" break _apply_body_format(paragraph, W_NS) document_out = ET.tostring( root, encoding="utf-8", xml_declaration=True ) with zipfile.ZipFile(source, "r") as source_zip: with zipfile.ZipFile( target, "w", compression=zipfile.ZIP_STORED ) as target_zip: for info in source_zip.infolist(): data = ( document_out if info.filename == document_name and document_out is not None else source_zip.read(info.filename) ) target_zip.writestr(info, data) def _collect_redundant_actions(report, chapter_id): remove_keys = set() fills = [] if report is None: return remove_keys, fills for issue in getattr(report, "issues", []) or []: if issue.issue_type != "redundant_info" or issue.chapter_id != chapter_id: continue try: payload = json.loads(issue.suggestion or "{}") except Exception: continue for item in payload.get("removals", []) or []: if isinstance(item, dict) and item.get("text"): raw = str(item["text"]).strip() if _EMPTY_LABEL_RE.match(raw): remove_keys.add(_normalize_docx_text(raw)) for item in payload.get("fills", []) or []: if isinstance(item, dict): fills.append(item) return remove_keys, fills def _write_step5_chapters_from_step4(records, outline, analysis, base_dir, report=None): """以 step4 章节 DOCX 为底稿,在其上应用 Step5 修复并写出 step5_chapters。""" chapter_dir = os.path.join( base_dir, "chapters", _safe_project_name(outline.project_name) ) os.makedirs(chapter_dir, exist_ok=True) has_packages = _has_package_info(analysis) manifest = [] for record in records: source = record.get("artifact_path", "") chapter_id = str(record.get("id", "")) if not source or not os.path.isfile(source): continue target = os.path.join(chapter_dir, os.path.basename(source)) remove_keys, fills = _collect_redundant_actions(report, chapter_id) _safe_copy_and_clean_chapter( source, target, has_packages, remove_keys, fills ) manifest.append({ "id": chapter_id, "title": record.get("title", ""), "artifact_path": os.path.abspath(target), "status": record.get("status", "complete"), }) manifest_path = os.path.join(chapter_dir, "manifest.json") with open(manifest_path, "w", encoding="utf-8") as file: json.dump( { "project_name": outline.project_name, "expected_count": len(records), "completed_count": len(manifest), "chapters": manifest, }, file, ensure_ascii=False, indent=2, ) return [record["artifact_path"] for record in manifest] def main(): analysis, project_data = _load_step2_info(STEP2_INFO_FILE) report = _load_step3_report(STEP3_REPORT_FILE) step4_records = _load_step4_records(STEP4_CHAPTERS_DIR) chapter_texts = _load_chapter_texts(STEP4_CHAPTERS_DIR) outline = _build_outline_from_report(report, chapter_texts) print("=" * 60) print("Step 5a: 加载实际章节内容、Step3 大纲与评分/废标映射") print("=" * 60) print(f"章节数: {len(outline.chapters)}") print(f"评分项: {len(analysis.scoring_criteria)};废标项: {len(analysis.rejection_items)}") print(f"Step3 评分/废标索引映射: {len(outline.evaluation_index_entries)} 条") print(f"Step3 标题关系映射: {len(outline.heading_mappings)} 条") print(f"已读取实际章节文件: {len(chapter_texts)} 个") for chapter in outline.chapters: print( f" {chapter.id} {chapter.title}: 实际正文 " f"{len(chapter.generated_content or '')} 字符" ) print() print("=" * 60) print("Step 5b: 内容审核") print("=" * 60) from step5_reviewing import review_content, auto_fix_issues review_report, outline = review_content(outline, analysis) print() print("=== 审核报告 ===") print(f"审核结果: {'[PASS] 通过' if review_report.passed else '[FAIL] 未通过'}") print(f"总字数: {review_report.total_word_count:,}") print(f"问题总数: {len(review_report.issues)}") print( "问题类型分布:", dict(Counter(i.issue_type for i in review_report.issues)), ) errors = [i for i in review_report.issues if i.severity == "error"] warnings = [i for i in review_report.issues if i.severity == "warning"] print(f" 错误: {len(errors)}") for issue in errors: print(f" [ERR] [{issue.chapter_id}] {issue.description[:120]}") if issue.suggestion: print(f" 建议: {issue.suggestion[:120]}") print(f" 警告: {len(warnings)}") for issue in warnings[:8]: print(f" [WARN] [{issue.chapter_id}] {issue.description[:120]}") if not review_report.passed: print() print("=" * 60) print("Step 5c: 自动修复") print("=" * 60) outline = auto_fix_issues( outline, analysis, review_report, project_data=project_data, ) report2, outline = review_content(outline, analysis) print(f"\n修复后审核: {'[PASS] 通过' if report2.passed else '[FAIL] 仍有问题'}") print(f"修复后字数: {report2.total_word_count:,}") print(f"剩余问题: {len(report2.issues)}") print() print("=" * 60) print("Step 5d: 在 step4 各章 DOCX 基础上写出 step5_chapters") print("=" * 60) artifact_paths = _write_step5_chapters_from_step4( step4_records, outline, analysis, STEP5_CHAPTERS_DIR, report=review_report, ) print(f"已写出 {len(artifact_paths)} 个章节 DOCX 到: {STEP5_CHAPTERS_DIR}") os.makedirs(os.path.dirname(STEP5_OUTLINE_FILE), exist_ok=True) with open(STEP5_OUTLINE_FILE, "wb") as file: pickle.dump(outline, file) print(f"\nStep5 修复后大纲已保存: {STEP5_OUTLINE_FILE}") print("Step 5 测试完成 [OK]") if __name__ == "__main__": main()