proposa-ai/run_pipeline.py

"""
Pipeline 入口 — 按阶段运行投标书生成流程

用法:
    python run_pipeline.py --phase 1                                # 使用默认路径
    python run_pipeline.py --phase 1 --tender data/招标.pdf         # 指定新招标
    python run_pipeline.py --phase 1 --history data/history/        # 指定历史数据目录
    python run_pipeline.py --phase 2
    python run_pipeline.py --phase 1-3
    python run_pipeline.py --list
"""

import sys
import os
import json
import glob

sys.stdout.reconfigure(encoding="utf-8")
BASE_DIR = os.path.dirname(os.path.abspath(__file__))


# ── 阶段配置 ──

PHASES = {
    1: {
        "name": "phase1_parsing",
        "title": "招标解析与历史比对",
        "desc": "输入新招标PDF + 历史招投标数据目录 → 输出招标信息、表格、历史浓缩及相似度权重",
        "outputs": [
            "output/phase1_new_tender.json",
            "output/phase1_tender_tables/tables.json",
            "output/phase1_historical_summary.json",
        ],
    },
    2: {
        "name": "phase2_outline",
        "title": "投标大纲生成",
        "desc": "基于新招标要求和加权后的历史参考，用 LLM 生成目录大纲",
        "outputs": ["output/phase2_bid_outline.json"],
        "deps": [1],
    },
    3: {
        "name": "phase3_collection",
        "title": "公司信息模板生成",
        "desc": "生成投标公司需要补充的信息清单（简化版 JSON）",
        "outputs": ["output/phase3_company_template.json"],
        "deps": [1],
    },
    4: {
        "name": "phase4_filling",
        "title": "内容填充（待开发）",
        "outputs": ["output/phase4_bid_fill_draft.json"],
        "deps": [2, 3],
    },
    5: {
        "name": "phase5_validation",
        "title": "一致性校验（待开发）",
        "outputs": ["output/phase5_consistency_report.json"],
        "deps": [4],
    },
    6: {
        "name": "phase6_export",
        "title": "DOCX 导出（待开发）",
        "outputs": ["output/phase6_bid_final.docx"],
        "deps": [4, 5],
    },
}


# ── 阶段执行 ──

def run_phase(phase_num: int, tender: str = None, history: str = None):
    info = PHASES[phase_num]
    print(f"\n{'='*60}")
    print(f"Phase {phase_num}: {info['title']}")
    print(f"{'='*60}")
    print(f"说明: {info['desc']}")

    for dep in info.get("deps", []):
        for out_path in PHASES[dep]["outputs"]:
            if not os.path.exists(os.path.join(BASE_DIR, out_path)):
                print(f"  [WARN] 依赖 Phase {dep} 输出不存在: {out_path}")

    if phase_num == 1:
        _run_phase1(tender, history)
    elif phase_num == 2:
        _run_phase2()
    elif phase_num == 3:
        _run_phase3()
    else:
        print(f"  [INFO] Phase {phase_num} 尚未开发")


# ════════════════════════════════════════════════════════════
# Phase 1: 新招标解析 + 历史招投标比对
# ════════════════════════════════════════════════════════════

def _run_phase1(tender_path: str = None, history_dir: str = None):
    """
    输入:
      - 新招标文件 PDF（1 份）
      - 历史数据目录（N 组），每组结构:
          项目文件夹/
          ├── 招标文件.pdf    ← 历史招标（用于和新招标算相似度）
          └── 投标文件.pdf    ← 对应的投标（按相似度加权影响新投标）

    输出:
      - output/phase1_new_tender.json          新招标解析结果（文本+特征+表格索引）
      - output/phase1_tender_tables/tables.json 表格元数据
      - output/phase1_historical_summary.json   历史数据汇总（含相似度权重+投标结构）
    """
    from phase1_parsing.extract_pdf_text import extract_text, extract_tender_sections, extract_project_overview
    from phase1_parsing.extract_pdf_tables import extract_tables
    from phase1_parsing.analyze_similarity import calculate_similarity

    # ── 1. 解析新招标文件 ────────────────────────────

    tender_pdf = tender_path or "data/松江区机关事务管理局物业管理服务招标文件.pdf"
    if not os.path.exists(tender_pdf):
        print(f"[ERROR] 新招标文件不存在: {tender_pdf}")
        return

    print(f"\n[Phase1] 解析新招标文件: {tender_pdf}")
    tender_text = extract_text(tender_pdf, "output/phase1_raw/tender_text.txt")

    # 章节分割
    sections = extract_tender_sections(tender_text)
    for name, content in sections.items():
        print(f"  章节 {name}: {len(content)} 字符")

    # 表格提取
    print("\n  提取表格...")
    try:
        tables_meta = extract_tables(tender_pdf, "output/phase1_tender_tables")
        table_count = tables_meta.get("total_tables", 0)
        print(f"  共 {table_count} 个表格")
    except Exception as e:
        print(f"  [SKIP] 表格提取跳过: {e}")
        table_count = 0

    # 项目概况
    overview = extract_project_overview(tender_text)
    print(f"  项目概况: {overview.get('project_name', '未知')}")

    # 保存新招标解析
    new_tender_data = {
        "source_file": tender_pdf,
        "text_file": "output/phase1_raw/tender_text.txt",
        "total_chars": len(tender_text),
        "sections": {k: len(v) for k, v in sections.items()},
        "tables_count": table_count,
        "overview": overview,
    }
    os.makedirs("output", exist_ok=True)
    with open("output/phase1_new_tender.json", "w", encoding="utf-8") as f:
        json.dump(new_tender_data, f, ensure_ascii=False, indent=2)
    print("  [OK] 已保存: output/phase1_new_tender.json")

    # ── 2. 扫描历史数据 ─────────────────────────────

    history_dir = history_dir or "data/历史数据"
    if not os.path.isdir(history_dir):
        print(f"\n[Phase1] 历史数据目录不存在: {history_dir}")
        print("  将仅输出新招标解析结果")
        return

    groups = _scan_history_groups(history_dir)
    if not groups:
        print(f"\n[Phase1] 在 {history_dir} 下未找到历史数据")
        return

    print(f"\n[Phase1] 历史数据: 发现 {len(groups)} 组")
    for g in groups:
        print(f"  [{g['group_id']}] {g.get('name', '')}")

    # ── 3. 相似度计算 ───────────────────────────────

    print("\n[Phase1] 计算历史招标与新招标的相似度...")
    similarity_results = calculate_similarity(tender_text, groups, use_llm=True)

    print("\n  相似度排名:")
    for r in similarity_results:
        bar = "█" * int(r["similarity_score"] * 20) + "░" * (20 - int(r["similarity_score"] * 20))
        print(f"  {r['similarity_score']:.0%} {bar}  {r['name']} ({r['similarity_level']})")
        print(f"     理由: {r['similarity_reason'][:80]}")

    # ── 4. 输出历史汇总 ─────────────────────────────

    summary = {
        "new_tender": {
            "file": tender_pdf,
            "overview": overview,
            "text_file": "output/phase1_raw/tender_text.txt",
        },
        "historical_count": len(groups),
        "historical_data": similarity_results,
        "note": "每项含 similarity_score(0~1) 表示与新招标的相似度，weight 为归一化权重。"
                "相似度越高，对应的投标文件对结果影响越大。",
    }
    with open("output/phase1_historical_summary.json", "w", encoding="utf-8") as f:
        json.dump(summary, f, ensure_ascii=False, indent=2)
    print(f"\n[Phase1] 历史汇总已保存: output/phase1_historical_summary.json")


def _scan_history_groups(history_dir: str) -> list:
    """
    扫描历史数据目录。支持两种结构:

    结构1: 项目文件夹模式（推荐）
        history_dir/
        ├── 项目A/
        │   ├── 招标文件.pdf
        │   └── 投标文件.pdf
        ├── 项目B/
        │   ├── 招标文件.pdf
        │   └── 投标文件.pdf

    结构2: 平铺模式（自动配对）
        history_dir/
        ├── XX项目-招标文件.pdf
        ├── XX项目-投标文件.pdf
        ├── YY项目-招标文件.pdf
        └── YY项目-投标文件.pdf
    """
    from phase1_parsing.extract_pdf_text import extract_text

    groups = []
    seen = set()

    # 结构1: 子文件夹模式
    for sub_dir in sorted(glob.glob(os.path.join(history_dir, "*"))):
        if not os.path.isdir(sub_dir):
            continue
        tender_file = _find_file(sub_dir, "*招标*.pdf") or _find_file(sub_dir, "*tender*.pdf")
        bid_file = _find_file(sub_dir, "*投标*.pdf") or _find_file(sub_dir, "*bid*.pdf")
        if tender_file and bid_file:
            group_id = os.path.basename(sub_dir)
            print(f"  [扫描] {group_id}: 招标={os.path.basename(tender_file)}, 投标={os.path.basename(bid_file)}")
            tender_text = extract_text(tender_file)
            bid_text = extract_text(bid_file)
            groups.append({
                "group_id": group_id,
                "name": group_id,
                "tender_file": tender_file,
                "bid_file": bid_file,
                "tender_text": tender_text,
                "bid_text": bid_text,
            })
            seen.add(sub_dir)

    # 结构2: 平铺自动配对
    all_pdfs = sorted(glob.glob(os.path.join(history_dir, "*.pdf")))
    pending = []
    for fp in all_pdfs:
        fname = os.path.basename(fp)
        if fp in seen:
            continue
        if "招标" in fname or "tender" in fname.lower():
            pending.append(("tender", fp))
        elif "投标" in fname or "bid" in fname.lower():
            pending.append(("bid", fp))

    # 简单配对: 按顺序招标+投标为一组
    i = 0
    while i < len(pending) - 1:
        if pending[i][0] == "tender" and pending[i + 1][0] == "bid":
            tender_file = pending[i][1]
            bid_file = pending[i + 1][1]
            group_id = f"history_group_{len(groups) + 1}"
            print(f"  [扫描] {group_id}: 招标={os.path.basename(tender_file)}, 投标={os.path.basename(bid_file)}")
            tender_text = extract_text(tender_file)
            bid_text = extract_text(bid_file)
            groups.append({
                "group_id": group_id,
                "name": group_id,
                "tender_file": tender_file,
                "bid_file": bid_file,
                "tender_text": tender_text,
                "bid_text": bid_text,
            })
            i += 2
        else:
            i += 1

    return groups


def _find_file(directory: str, pattern: str) -> str:
    """在目录中匹配第一个文件"""
    files = glob.glob(os.path.join(directory, pattern))
    return files[0] if files else None


# ════════════════════════════════════════════════════════════
# Phase 2: 投标大纲生成
# ════════════════════════════════════════════════════════════

def _run_phase2():
    """读取 Phase1 输出 → 生成目录大纲（历史相似度高的优先参考）"""
    from phase2_outline.generate_outline import generate_outline, print_outline_preview

    new_tender_file = "output/phase1_new_tender.json"
    hist_file = "output/phase1_historical_summary.json"

    if not os.path.exists(new_tender_file):
        print(f"[ERROR] 请先运行 Phase 1: 未找到 {new_tender_file}")
        return

    # 读取新招标文本
    with open(new_tender_file, "r", encoding="utf-8") as f:
        new_tender = json.load(f)

    tender_text_file = new_tender.get("text_file", "output/phase1_raw/tender_text.txt")
    if not os.path.exists(tender_text_file):
        print(f"[ERROR] 招标文本不存在: {tender_text_file}")
        return

    with open(tender_text_file, "r", encoding="utf-8") as f:
        tender_text = f.read()

    # 读取历史汇总
    bid_toc = []
    if os.path.exists(hist_file):
        with open(hist_file, "r", encoding="utf-8") as f:
            hist_data = json.load(f)

        # 取相似度最高的历史投标结构作为参考
        hist_items = hist_data.get("historical_data", [])
        if hist_items:
            best = hist_items[0]
            bid_toc = best.get("bid_toc", [])
            print(f"\n[Phase2] 参考历史: {best.get('name', 'N/A')} "
                  f"(相似度 {best.get('similarity_score', 0):.0%}, 权重 {best.get('weight', 0):.0%})")

    print(f"\n[Phase2] 招标文本: {len(tender_text)} 字符")
    print(f"[Phase2] 历史投标目录: {len(bid_toc)} 行")

    outline = generate_outline(tender_text, bid_toc,
                               output_path="output/phase2_bid_outline.json")
    print_outline_preview(outline)


# ════════════════════════════════════════════════════════════
# Phase 3: 公司信息模板生成
# ════════════════════════════════════════════════════════════

def _run_phase3():
    from phase3_collection.generate_template import generate_template
    template = generate_template(output_path="output/phase3_company_template.json")
    items = template["投标公司信息收集表"]["信息项"]
    print(f"\n共 {len(items)} 项信息待补充:")
    for item in items:
        print(f"  □ {item['名称']}")


# ── CLI ──

def list_phases():
    print("\nPipeline 阶段:\n")
    for num in sorted(PHASES):
        info = PHASES[num]
        deps = f" (依赖: Phase {', '.join(str(d) for d in info['deps'])})" if info.get("deps") else ""
        print(f"  Phase {num}: {info['title']}{deps}")
        print(f"            {info['desc']}")
        for o in info["outputs"]:
            fp = os.path.join(BASE_DIR, o)
            status = "✓" if os.path.exists(fp) else " "
            print(f"            [{status}] {o}")
        print()


def main():
    import argparse
    parser = argparse.ArgumentParser(description="投标书生成 Pipeline")
    parser.add_argument("--phase", help="运行阶段: 1, 2, 1-3")
    parser.add_argument("--all", action="store_true", help="全流程")
    parser.add_argument("--list", action="store_true", help="查看阶段说明")
    parser.add_argument("--tender", default=None, help="新招标文件 PDF 路径")
    parser.add_argument("--history", default=None, help="历史数据目录（含多组招投标文件夹）")
    args = parser.parse_args()

    if args.list:
        list_phases()
        return

    phases_to_run = []
    if args.all:
        phases_to_run = sorted(PHASES.keys())
    elif args.phase:
        for part in args.phase.split(","):
            if "-" in part:
                s, e = part.split("-")
                phases_to_run.extend(range(int(s), int(e) + 1))
            else:
                phases_to_run.append(int(part))

    if not phases_to_run:
        parser.print_help()
        list_phases()
        return

    print("执行顺序:", " → ".join(f"Phase {p}" for p in phases_to_run))
    for p in phases_to_run:
        if p in PHASES:
            run_phase(p, tender=args.tender, history=args.history)
        else:
            print(f"[ERROR] 未知阶段: Phase {p}")


if __name__ == "__main__":
    main()