feat: HSAP platform v2 — modular navigation, quality review, audit log, world model simulation
Major changes: - New frontend (platform/web/): Vite + React 18 + TypeScript + Tailwind - 4-module navigation: 数据送标 / 模型管理 / 车队管理 / 系统管理 - Data catalog with charts (DMS/ADAS/Lane 3-tab view) - Quality review workflow (标注质检): Good/Fine/Bad scoring with auto-advance - Audit enhancements: batch operations, rejection categories, Feishu notifications - Operation audit log (操作日志) - World model simulation studio (仿真工坊) - Dataset version management with snapshots and diff - ADAS 7-class dataset integration (138K images organized + compressed) - User management with Feishu integration and pagination - CRUD/search/filter on all pages, card layout redesign - PIL-optimized image overlay rendering - Auto-snapshot on build, in_review workflow stage - Removed embedded algorithm code (now in workspace)
This commit is contained in:
1
platform/as_platform/integrations/__init__.py
Normal file
1
platform/as_platform/integrations/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""外部系统集成。"""
|
||||
120
platform/as_platform/integrations/delivery_ingest.py
Normal file
120
platform/as_platform/integrations/delivery_ingest.py
Normal file
@@ -0,0 +1,120 @@
|
||||
"""平台送标申请审批通过后：analyze → promote 入湖。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.data.lake import (
|
||||
analyze_directory_candidate,
|
||||
create_directory_candidate,
|
||||
promote_candidate_to_inbox,
|
||||
)
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import BatchDelivery
|
||||
|
||||
|
||||
def validate_delivery_fields(
|
||||
*,
|
||||
project: str,
|
||||
task: str | None,
|
||||
mode: str | None,
|
||||
batch_name: str,
|
||||
data_path: str,
|
||||
) -> str | None:
|
||||
if not project or not batch_name:
|
||||
return "缺少 项目 或 批次名"
|
||||
if batch_name == (task or ""):
|
||||
return "批次名不能与任务名相同"
|
||||
if task in ("dam", "forward") and not (mode or "").strip():
|
||||
return f"任务 {task} 须填写 子模式"
|
||||
path = (data_path or "").strip()
|
||||
if not path:
|
||||
return "须填写 数据路径"
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return f"数据路径不存在: {path}"
|
||||
if project == "dms" and p.name in ("train", "val", "test") and p.parent.name == "images":
|
||||
return f"请填批次根目录（例如 {p.parent.parent}），不要填到 images/train"
|
||||
return None
|
||||
|
||||
|
||||
def ingest_from_directory(
|
||||
*,
|
||||
project: str,
|
||||
task: str | None,
|
||||
mode: str | None,
|
||||
batch_name: str,
|
||||
data_path: str,
|
||||
delivery_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""目录分析并 promote 到 inbox。"""
|
||||
err = validate_delivery_fields(
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
batch_name=batch_name,
|
||||
data_path=data_path,
|
||||
)
|
||||
if err:
|
||||
raise ValueError(err)
|
||||
|
||||
src = Path(data_path.strip())
|
||||
task_eff = task if project == "dms" else None
|
||||
cand = create_directory_candidate(
|
||||
project=project,
|
||||
task=task_eff,
|
||||
mode=mode or None,
|
||||
source_dir=src,
|
||||
source_type="platform_delivery",
|
||||
external_id=delivery_id,
|
||||
)
|
||||
cid = cand["id"]
|
||||
analyze_directory_candidate(cid, src)
|
||||
promo = promote_candidate_to_inbox(cid, batch=batch_name, mode=mode or None)
|
||||
return {
|
||||
"ok": True,
|
||||
"candidate_id": cid,
|
||||
"inbox_path": promo.get("inbox_path") or "",
|
||||
"batch": promo.get("batch") or batch_name,
|
||||
}
|
||||
|
||||
|
||||
def run_delivery_ingest(delivery_id: str) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchDelivery, delivery_id)
|
||||
if not rec:
|
||||
raise ValueError(f"送标申请不存在: {delivery_id}")
|
||||
rec.status = "ingesting"
|
||||
rec.error_message = None
|
||||
db.flush()
|
||||
project = rec.project
|
||||
task = rec.task
|
||||
mode = rec.mode
|
||||
batch_name = rec.batch_name
|
||||
data_path = rec.data_path
|
||||
|
||||
try:
|
||||
result = ingest_from_directory(
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
batch_name=batch_name,
|
||||
data_path=data_path,
|
||||
delivery_id=delivery_id,
|
||||
)
|
||||
except Exception as e:
|
||||
from as_platform.deliveries.service import mark_delivery_ingest_failed
|
||||
|
||||
mark_delivery_ingest_failed(delivery_id, None, str(e))
|
||||
raise
|
||||
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchDelivery, delivery_id)
|
||||
if rec:
|
||||
rec.status = "in_lake"
|
||||
rec.candidate_id = result.get("candidate_id")
|
||||
rec.inbox_path = result.get("inbox_path")
|
||||
rec.error_message = None
|
||||
db.flush()
|
||||
|
||||
return result
|
||||
299
platform/as_platform/integrations/feishu_bitable.py
Normal file
299
platform/as_platform/integrations/feishu_bitable.py
Normal file
@@ -0,0 +1,299 @@
|
||||
"""飞书多维表格 API（tenant_access_token）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from as_platform.auth.feishu import _get_tenant_access_token
|
||||
from as_platform.config import (
|
||||
FEISHU_APP_ID,
|
||||
FEISHU_APP_SECRET,
|
||||
FEISHU_BITABLE_APP_TOKEN,
|
||||
FEISHU_BITABLE_FIELDS,
|
||||
FEISHU_BITABLE_TABLE_ID,
|
||||
FEISHU_BITABLE_WIKI_NODE_TOKEN,
|
||||
)
|
||||
|
||||
BITABLE_BASE = "https://open.feishu.cn/open-apis/bitable/v1"
|
||||
WIKI_GET_NODE = "https://open.feishu.cn/open-apis/wiki/v2/spaces/get_node"
|
||||
|
||||
_field_cache: dict[str, Any] | None = None
|
||||
_field_cache_at: float = 0.0
|
||||
_FIELD_CACHE_TTL = 300.0
|
||||
_resolved_app_token: str | None = None
|
||||
|
||||
|
||||
def _looks_like_bitable_app_token(token: str) -> bool:
|
||||
t = token.strip()
|
||||
return t.startswith(("Basc", "basc", "app"))
|
||||
|
||||
|
||||
def _resolve_app_token_from_wiki(node_token: str) -> str:
|
||||
with _client() as client:
|
||||
access = _token(client)
|
||||
resp = client.get(
|
||||
WIKI_GET_NODE,
|
||||
headers={"Authorization": f"Bearer {access}"},
|
||||
params={"token": node_token},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "wiki get_node failed")
|
||||
node = data.get("data", {}).get("node") or {}
|
||||
obj = (node.get("obj_token") or "").strip()
|
||||
if not obj:
|
||||
raise RuntimeError(
|
||||
"wiki 节点未返回 obj_token；请在开放平台开通 wiki:node:read，或改用 /base/Basc... 填 FEISHU_BITABLE_APP_TOKEN"
|
||||
)
|
||||
return obj
|
||||
|
||||
|
||||
def _effective_app_token() -> str:
|
||||
global _resolved_app_token
|
||||
if FEISHU_BITABLE_APP_TOKEN and _looks_like_bitable_app_token(FEISHU_BITABLE_APP_TOKEN):
|
||||
return FEISHU_BITABLE_APP_TOKEN
|
||||
node = FEISHU_BITABLE_WIKI_NODE_TOKEN or FEISHU_BITABLE_APP_TOKEN
|
||||
if not node:
|
||||
return FEISHU_BITABLE_APP_TOKEN
|
||||
if _resolved_app_token is None:
|
||||
_resolved_app_token = _resolve_app_token_from_wiki(node)
|
||||
return _resolved_app_token
|
||||
|
||||
|
||||
def is_bitable_configured() -> bool:
|
||||
has_app = bool(FEISHU_BITABLE_APP_TOKEN or FEISHU_BITABLE_WIKI_NODE_TOKEN)
|
||||
return bool(FEISHU_APP_ID and FEISHU_APP_SECRET and has_app and FEISHU_BITABLE_TABLE_ID)
|
||||
|
||||
|
||||
def _client() -> httpx.Client:
|
||||
return httpx.Client(timeout=60.0)
|
||||
|
||||
|
||||
def _token(client: httpx.Client) -> str:
|
||||
return _get_tenant_access_token(client)
|
||||
|
||||
|
||||
def _app_table() -> tuple[str, str]:
|
||||
return _effective_app_token(), FEISHU_BITABLE_TABLE_ID
|
||||
|
||||
|
||||
def list_tables() -> list[dict[str, Any]]:
|
||||
"""列出 Base 下数据表（运维查 table_id）。"""
|
||||
if not is_bitable_configured():
|
||||
return []
|
||||
app_token, _ = _app_table()
|
||||
with _client() as client:
|
||||
token = _token(client)
|
||||
resp = client.get(
|
||||
f"{BITABLE_BASE}/apps/{app_token}/tables",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
params={"page_size": 100},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "list tables failed")
|
||||
return data.get("data", {}).get("items") or []
|
||||
|
||||
|
||||
def _load_field_meta(force: bool = False) -> dict[str, dict[str, Any]]:
|
||||
global _field_cache, _field_cache_at
|
||||
now = time.time()
|
||||
if not force and _field_cache and now - _field_cache_at < _FIELD_CACHE_TTL:
|
||||
return _field_cache
|
||||
|
||||
app_token, table_id = _app_table()
|
||||
by_name: dict[str, dict[str, Any]] = {}
|
||||
with _client() as client:
|
||||
token = _token(client)
|
||||
page_token: str | None = None
|
||||
while True:
|
||||
params: dict[str, Any] = {"page_size": 100}
|
||||
if page_token:
|
||||
params["page_token"] = page_token
|
||||
resp = client.get(
|
||||
f"{BITABLE_BASE}/apps/{app_token}/tables/{table_id}/fields",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
params=params,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "list fields failed")
|
||||
for item in data.get("data", {}).get("items") or []:
|
||||
name = item.get("field_name") or item.get("name")
|
||||
if name:
|
||||
by_name[str(name)] = item
|
||||
page_token = data.get("data", {}).get("page_token")
|
||||
if not page_token:
|
||||
break
|
||||
_field_cache = by_name
|
||||
_field_cache_at = now
|
||||
return by_name
|
||||
|
||||
|
||||
def _option_id_for_text(field_meta: dict[str, Any], text: str) -> str | None:
|
||||
ui = field_meta.get("ui_type") or field_meta.get("type")
|
||||
if ui not in (3, "SingleSelect", "single_select"):
|
||||
return None
|
||||
prop = field_meta.get("property") or {}
|
||||
for opt in prop.get("options") or []:
|
||||
if opt.get("name") == text:
|
||||
return opt.get("id")
|
||||
return None
|
||||
|
||||
|
||||
def _encode_value(field_meta: dict[str, Any], value: Any) -> Any:
|
||||
if value is None:
|
||||
return None
|
||||
ui = field_meta.get("ui_type") or field_meta.get("type")
|
||||
if ui in (1, "Text", "text", "Url", "url"):
|
||||
if isinstance(value, dict) and "link" in value:
|
||||
return value
|
||||
return str(value)
|
||||
if ui in (2, "Number", "number"):
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if ui in (3, "SingleSelect", "single_select"):
|
||||
oid = _option_id_for_text(field_meta, str(value))
|
||||
return oid if oid else str(value)
|
||||
if ui in (5, "DateTime", "datetime", "CreatedTime", "ModifiedTime"):
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
if isinstance(value, datetime):
|
||||
dt = value if value.tzinfo else value.replace(tzinfo=timezone.utc)
|
||||
return int(dt.timestamp() * 1000)
|
||||
return int(datetime.now(timezone.utc).timestamp() * 1000)
|
||||
if ui in (15, "Url", "url") and isinstance(value, dict):
|
||||
return {"text": value.get("text") or value.get("link"), "link": value.get("link")}
|
||||
return str(value)
|
||||
|
||||
|
||||
def _flatten_cell(cell: Any) -> str:
|
||||
if cell is None:
|
||||
return ""
|
||||
if isinstance(cell, str):
|
||||
return cell.strip()
|
||||
if isinstance(cell, (int, float)):
|
||||
return str(cell)
|
||||
if isinstance(cell, list):
|
||||
parts = [_flatten_cell(x) for x in cell]
|
||||
return ",".join(p for p in parts if p)
|
||||
if isinstance(cell, dict):
|
||||
if "text" in cell:
|
||||
return str(cell.get("text") or "").strip()
|
||||
if "name" in cell:
|
||||
return str(cell.get("name") or "").strip()
|
||||
if "value" in cell:
|
||||
return _flatten_cell(cell.get("value"))
|
||||
return str(cell).strip()
|
||||
|
||||
|
||||
def list_all_records() -> list[dict[str, Any]]:
|
||||
"""返回 {record_id, fields_raw, flat}。"""
|
||||
if not is_bitable_configured():
|
||||
return []
|
||||
app_token, table_id = _app_table()
|
||||
meta = _load_field_meta()
|
||||
id_by_label = {label: meta[name]["field_id"] for name, m in meta.items() if (label := FEISHU_BITABLE_FIELDS.get(name)) and name in meta for name in FEISHU_BITABLE_FIELDS}
|
||||
# rebuild: map logical key -> field_name
|
||||
name_for_key = {k: v for k, v in FEISHU_BITABLE_FIELDS.items()}
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
with _client() as client:
|
||||
token = _token(client)
|
||||
page_token: str | None = None
|
||||
while True:
|
||||
body: dict[str, Any] = {"page_size": 500}
|
||||
if page_token:
|
||||
body["page_token"] = page_token
|
||||
resp = client.post(
|
||||
f"{BITABLE_BASE}/apps/{app_token}/tables/{table_id}/records/search",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json=body,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "search records failed")
|
||||
for item in data.get("data", {}).get("items") or []:
|
||||
record_id = item.get("record_id") or item.get("id")
|
||||
fields = item.get("fields") or {}
|
||||
flat: dict[str, str] = {}
|
||||
for key, col_name in name_for_key.items():
|
||||
raw = fields.get(col_name)
|
||||
if raw is None:
|
||||
fid = meta.get(col_name, {}).get("field_id")
|
||||
if fid:
|
||||
raw = fields.get(fid)
|
||||
flat[key] = _flatten_cell(raw)
|
||||
rows.append({"record_id": record_id, "fields": fields, "flat": flat})
|
||||
page_token = data.get("data", {}).get("page_token")
|
||||
if not page_token:
|
||||
break
|
||||
return rows
|
||||
|
||||
|
||||
def update_record(record_id: str, values: dict[str, Any]) -> None:
|
||||
"""values: 逻辑键 delivery_id / status / inbox_path ..."""
|
||||
if not record_id or not is_bitable_configured():
|
||||
return
|
||||
meta = _load_field_meta()
|
||||
payload_fields: dict[str, Any] = {}
|
||||
for key, val in values.items():
|
||||
if val is None:
|
||||
continue
|
||||
col = FEISHU_BITABLE_FIELDS.get(key)
|
||||
if not col or col not in meta:
|
||||
continue
|
||||
encoded = _encode_value(meta[col], val)
|
||||
if encoded is not None:
|
||||
payload_fields[col] = encoded
|
||||
|
||||
if not payload_fields:
|
||||
return
|
||||
|
||||
app_token, table_id = _app_table()
|
||||
with _client() as client:
|
||||
token = _token(client)
|
||||
resp = client.put(
|
||||
f"{BITABLE_BASE}/apps/{app_token}/tables/{table_id}/records/{record_id}",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json={"fields": payload_fields},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "update record failed")
|
||||
|
||||
|
||||
def connectivity_check() -> dict[str, Any]:
|
||||
if not is_bitable_configured():
|
||||
return {"configured": False, "ok": False, "message": "缺少 FEISHU_BITABLE_APP_TOKEN 或 TABLE_ID"}
|
||||
try:
|
||||
tables = list_tables()
|
||||
meta = _load_field_meta(force=True)
|
||||
missing = [v for k, v in FEISHU_BITABLE_FIELDS.items() if k not in ("record_id",) and v not in meta]
|
||||
return {
|
||||
"configured": True,
|
||||
"ok": len(missing) == 0,
|
||||
"tables": [{"table_id": t.get("table_id"), "name": t.get("name")} for t in tables],
|
||||
"field_count": len(meta),
|
||||
"missing_columns": missing,
|
||||
}
|
||||
except Exception as e:
|
||||
msg = str(e)
|
||||
hint = ""
|
||||
if "1254302" in msg or "no permissions" in msg.lower():
|
||||
hint = (
|
||||
"飞书返回 1254302：应用「主动安全算法平台」对该表无读写权。"
|
||||
"知识库内嵌表常无法加企业应用，请复制为独立多维表格(/base/Basc...)并加应用协作者，"
|
||||
"或见 docs/FEISHU_BITABLE_OPS.md §8；联调可设 FEISHU_BITABLE_SYNC_ENABLED=0。"
|
||||
)
|
||||
return {"configured": True, "ok": False, "message": msg, "hint": hint or None}
|
||||
158
platform/as_platform/integrations/feishu_bitable_ingest.py
Normal file
158
platform/as_platform/integrations/feishu_bitable_ingest.py
Normal file
@@ -0,0 +1,158 @@
|
||||
"""飞书多维表格「待落盘」→ analyze → promote（Phase B，FEISHU_BITABLE_AUTO_INGEST）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import FEISHU_BITABLE_FIELDS
|
||||
from as_platform.data.lake import (
|
||||
analyze_directory_candidate,
|
||||
create_feishu_directory_candidate,
|
||||
promote_candidate_to_inbox,
|
||||
)
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import DatasetCandidate, FeishuBitableLink
|
||||
from as_platform.integrations.feishu_bitable import is_bitable_configured, list_all_records, update_record
|
||||
from as_platform.integrations.feishu_bitable_sync import batch_key
|
||||
|
||||
|
||||
def _validate_row(flat: dict[str, str]) -> str | None:
|
||||
project = flat.get("project") or ""
|
||||
task = flat.get("task") or ""
|
||||
batch_name = flat.get("batch_name") or ""
|
||||
mode = flat.get("mode") or ""
|
||||
if not project or not batch_name:
|
||||
return "缺少 项目 或 批次名"
|
||||
if batch_name == task:
|
||||
return "批次名不能与任务名相同"
|
||||
if task in ("dam", "forward") and not mode:
|
||||
return f"任务 {task} 须填写 子模式"
|
||||
data_path = (flat.get("data_path") or "").strip()
|
||||
if not data_path:
|
||||
return "待落盘须填写 数据路径（内网 NAS）"
|
||||
if not Path(data_path).exists():
|
||||
return f"数据路径不存在: {data_path}"
|
||||
return None
|
||||
|
||||
|
||||
def _already_ingested(flat: dict[str, str], delivery_id: str) -> bool:
|
||||
cid = (flat.get("candidate_id") or "").strip()
|
||||
if cid:
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, cid)
|
||||
if rec and rec.status in ("promoted", "analyzed") and rec.inbox_path:
|
||||
return True
|
||||
if delivery_id:
|
||||
with session_scope() as db:
|
||||
link = db.query(FeishuBitableLink).filter_by(delivery_id=delivery_id).first()
|
||||
if link and link.inbox_path:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def process_pending_ingest() -> dict[str, Any]:
|
||||
if not is_bitable_configured():
|
||||
return {"ok": False, "message": "未配置多维表格", "processed": 0}
|
||||
|
||||
records = list_all_records()
|
||||
processed = 0
|
||||
skipped = 0
|
||||
errors: list[str] = []
|
||||
|
||||
for rec in records:
|
||||
record_id = rec.get("record_id")
|
||||
flat = rec.get("flat") or {}
|
||||
if not record_id:
|
||||
continue
|
||||
if flat.get("status") != "待落盘":
|
||||
continue
|
||||
|
||||
delivery_id = flat.get("delivery_id") or ""
|
||||
if _already_ingested(flat, delivery_id):
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
err = _validate_row(flat)
|
||||
if err:
|
||||
try:
|
||||
update_record(record_id, {"status": "落盘失败", "error_message": err})
|
||||
except Exception as e:
|
||||
errors.append(f"{record_id}: {e}")
|
||||
continue
|
||||
|
||||
try:
|
||||
update_record(record_id, {"status": "分析中", "error_message": ""})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
project = flat.get("project") or "dms"
|
||||
task = flat.get("task") or None
|
||||
mode = flat.get("mode") or None
|
||||
batch_name = flat.get("batch_name") or ""
|
||||
src = Path((flat.get("data_path") or "").strip())
|
||||
|
||||
try:
|
||||
cand = create_feishu_directory_candidate(
|
||||
project=project,
|
||||
task=task if project == "dms" else None,
|
||||
mode=mode or None,
|
||||
source_dir=src,
|
||||
external_id=delivery_id or None,
|
||||
feishu_record_id=record_id,
|
||||
)
|
||||
cid = cand["id"]
|
||||
analyze_directory_candidate(cid, src)
|
||||
promo = promote_candidate_to_inbox(cid, batch=batch_name, mode=mode or None)
|
||||
inbox_path = promo.get("inbox_path") or ""
|
||||
|
||||
update_record(
|
||||
record_id,
|
||||
{
|
||||
"status": "待送标",
|
||||
"candidate_id": cid,
|
||||
"inbox_path": inbox_path,
|
||||
"error_message": "",
|
||||
"record_id": record_id,
|
||||
},
|
||||
)
|
||||
|
||||
key = batch_key(
|
||||
project,
|
||||
task if project == "dms" else None,
|
||||
mode,
|
||||
promo.get("batch") or batch_name,
|
||||
)
|
||||
with session_scope() as db:
|
||||
link = db.query(FeishuBitableLink).filter_by(batch_key=key).first()
|
||||
if not link:
|
||||
link = FeishuBitableLink(
|
||||
batch_key=key,
|
||||
record_id=record_id,
|
||||
delivery_id=delivery_id or None,
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
batch=promo.get("batch") or batch_name,
|
||||
)
|
||||
db.add(link)
|
||||
link.record_id = record_id
|
||||
link.delivery_id = delivery_id or link.delivery_id
|
||||
link.inbox_path = inbox_path
|
||||
db.flush()
|
||||
|
||||
processed += 1
|
||||
except Exception as e:
|
||||
msg = str(e)
|
||||
errors.append(f"{record_id}: {msg}")
|
||||
try:
|
||||
update_record(record_id, {"status": "落盘失败", "error_message": msg[:500]})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"processed": processed,
|
||||
"skipped": skipped,
|
||||
"errors": errors[:20],
|
||||
"status_field": FEISHU_BITABLE_FIELDS.get("status"),
|
||||
}
|
||||
236
platform/as_platform/integrations/feishu_bitable_sync.py
Normal file
236
platform/as_platform/integrations/feishu_bitable_sync.py
Normal file
@@ -0,0 +1,236 @@
|
||||
"""HSAP 批次 → 飞书多维表格回写（内网 Phase A）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import FEISHU_STATUS_FROM_STAGE, FRONTEND_URL
|
||||
from as_platform.data.core import get_pending_report
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import FeishuBitableLink
|
||||
from as_platform.integrations.feishu_bitable import is_bitable_configured, list_all_records, update_record
|
||||
from as_platform.labeling.progress import campaign_progress_summary
|
||||
from as_platform.labeling.service import list_labeling_batches
|
||||
|
||||
|
||||
def batch_key(project: str, task: str | None, mode: str | None, batch: str, location: str = "inbox") -> str:
|
||||
return f"{location}:{project}:{task or ''}:{mode or ''}:{batch}"
|
||||
|
||||
|
||||
def _collect_hsap_batches() -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
try:
|
||||
for row in list_labeling_batches().get("items") or []:
|
||||
key = batch_key(
|
||||
row.get("project") or "dms",
|
||||
row.get("task"),
|
||||
row.get("mode"),
|
||||
row.get("batch") or "",
|
||||
row.get("location") or "inbox",
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
items.append(row)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
report = get_pending_report()
|
||||
for row in report.get("batches") or []:
|
||||
if row.get("stage") == "ingested":
|
||||
continue
|
||||
key = batch_key(
|
||||
row.get("project") or "dms",
|
||||
row.get("task"),
|
||||
row.get("mode"),
|
||||
row.get("batch") or "",
|
||||
row.get("location") or "inbox",
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
items.append(row)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _find_batch_for_record(
|
||||
flat: dict[str, str],
|
||||
hsap_batches: list[dict[str, Any]],
|
||||
links_by_delivery: dict[str, FeishuBitableLink],
|
||||
) -> dict[str, Any] | None:
|
||||
did = (flat.get("delivery_id") or "").strip()
|
||||
if did and did in links_by_delivery:
|
||||
key = links_by_delivery[did].batch_key
|
||||
for b in hsap_batches:
|
||||
if batch_key(
|
||||
b.get("project") or "dms",
|
||||
b.get("task"),
|
||||
b.get("mode"),
|
||||
b.get("batch") or "",
|
||||
b.get("location") or "inbox",
|
||||
) == key:
|
||||
return b
|
||||
for b in hsap_batches:
|
||||
if _match_record_to_batch(flat, b):
|
||||
return b
|
||||
return None
|
||||
|
||||
|
||||
def _match_record_to_batch(flat: dict[str, str], b: dict[str, Any]) -> bool:
|
||||
if flat.get("project") and flat.get("project") != (b.get("project") or ""):
|
||||
return False
|
||||
if flat.get("task") and flat.get("task") != (b.get("task") or ""):
|
||||
return False
|
||||
bm = flat.get("batch_name") or ""
|
||||
if bm and bm != (b.get("batch") or ""):
|
||||
return False
|
||||
fm = flat.get("mode") or ""
|
||||
if fm and fm != (b.get("mode") or ""):
|
||||
return False
|
||||
if not bm and not fm:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _progress_for_batch(b: dict[str, Any]) -> tuple[str, str | None]:
|
||||
cid = b.get("campaign_id")
|
||||
if not cid:
|
||||
return "", None
|
||||
try:
|
||||
prog = campaign_progress_summary(cid)
|
||||
return f"{prog.get('completed_tasks', 0)}/{prog.get('total_tasks', 0)}", cid
|
||||
except Exception:
|
||||
return "", cid
|
||||
|
||||
|
||||
def _hsap_link(b: dict[str, Any], campaign_id: str | None) -> dict[str, str]:
|
||||
base = FRONTEND_URL.rstrip("/")
|
||||
if campaign_id:
|
||||
path = f"/labeling/campaigns/{campaign_id}/annotate"
|
||||
label = "进入标注"
|
||||
else:
|
||||
path = "/labeling"
|
||||
label = "送标工作台"
|
||||
url = f"{base}{path}"
|
||||
batch = b.get("batch") or ""
|
||||
return {"text": f"{label} · {batch}", "link": url}
|
||||
|
||||
|
||||
def sync_hsap_to_bitable() -> dict[str, Any]:
|
||||
if not is_bitable_configured():
|
||||
return {"ok": False, "message": "飞书多维表格未配置", "updated": 0}
|
||||
|
||||
hsap_batches = _collect_hsap_batches()
|
||||
records = list_all_records()
|
||||
updated = 0
|
||||
matched_keys: set[str] = set()
|
||||
errors: list[str] = []
|
||||
|
||||
with session_scope() as db:
|
||||
links = db.query(FeishuBitableLink).all()
|
||||
links_by_delivery = {lnk.delivery_id: lnk for lnk in links if lnk.delivery_id}
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
for rec in records:
|
||||
record_id = rec.get("record_id")
|
||||
flat = rec.get("flat") or {}
|
||||
if not record_id:
|
||||
continue
|
||||
|
||||
hit = _find_batch_for_record(flat, hsap_batches, links_by_delivery)
|
||||
if not hit:
|
||||
continue
|
||||
|
||||
key = batch_key(
|
||||
hit.get("project") or "dms",
|
||||
hit.get("task"),
|
||||
hit.get("mode"),
|
||||
hit.get("batch") or "",
|
||||
hit.get("location") or "inbox",
|
||||
)
|
||||
matched_keys.add(key)
|
||||
|
||||
stage = hit.get("stage") or "raw_pool"
|
||||
feishu_status = FEISHU_STATUS_FROM_STAGE.get(stage)
|
||||
progress, cid = _progress_for_batch(hit)
|
||||
path = hit.get("path") or ""
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"record_id": record_id,
|
||||
"inbox_path": path,
|
||||
"progress": progress or None,
|
||||
"campaign_id": cid,
|
||||
"hsap_link": _hsap_link(hit, cid),
|
||||
"last_sync": now,
|
||||
}
|
||||
if feishu_status:
|
||||
payload["status"] = feishu_status
|
||||
|
||||
try:
|
||||
update_record(record_id, payload)
|
||||
updated += 1
|
||||
with session_scope() as db:
|
||||
link = db.query(FeishuBitableLink).filter_by(batch_key=key).first()
|
||||
if not link:
|
||||
link = FeishuBitableLink(
|
||||
batch_key=key,
|
||||
record_id=record_id,
|
||||
project=hit.get("project") or "dms",
|
||||
task=hit.get("task"),
|
||||
mode=hit.get("mode"),
|
||||
batch=hit.get("batch") or "",
|
||||
)
|
||||
db.add(link)
|
||||
link.record_id = record_id
|
||||
link.delivery_id = flat.get("delivery_id") or link.delivery_id
|
||||
link.campaign_id = cid
|
||||
link.inbox_path = path
|
||||
link.last_sync_at = now
|
||||
db.flush()
|
||||
except Exception as e:
|
||||
errors.append(f"{record_id}: {e}")
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"updated": updated,
|
||||
"hsap_batches": len(hsap_batches),
|
||||
"bitable_rows": len(records),
|
||||
"matched": len(matched_keys),
|
||||
"errors": errors[:20],
|
||||
}
|
||||
|
||||
|
||||
def backfill_hints() -> dict[str, Any]:
|
||||
"""返回尚未在飞书表匹配到的 HSAP 批次（需人工补录行）。"""
|
||||
hsap_batches = _collect_hsap_batches()
|
||||
records = list_all_records() if is_bitable_configured() else []
|
||||
unmatched: list[dict[str, Any]] = []
|
||||
|
||||
for b in hsap_batches:
|
||||
found = False
|
||||
for rec in records:
|
||||
if _match_record_to_batch(rec.get("flat") or {}, b):
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
unmatched.append(
|
||||
{
|
||||
"project": b.get("project"),
|
||||
"task": b.get("task"),
|
||||
"mode": b.get("mode"),
|
||||
"batch": b.get("batch"),
|
||||
"stage": b.get("stage"),
|
||||
"path": b.get("path"),
|
||||
"suggested_batch_name": b.get("batch"),
|
||||
}
|
||||
)
|
||||
|
||||
return {"unmatched_count": len(unmatched), "items": unmatched[:100]}
|
||||
68
platform/as_platform/integrations/feishu_notify.py
Normal file
68
platform/as_platform/integrations/feishu_notify.py
Normal file
@@ -0,0 +1,68 @@
|
||||
"""飞书群消息通知（出站，内网可用）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from as_platform.auth.feishu import _get_tenant_access_token
|
||||
from as_platform.config import FEISHU_APP_ID, FEISHU_APP_SECRET, FEISHU_LABELING_CHAT_ID
|
||||
|
||||
IM_MSG_URL = "https://open.feishu.cn/open-apis/im/v1/messages"
|
||||
|
||||
|
||||
def is_notify_configured() -> bool:
|
||||
return bool(FEISHU_APP_ID and FEISHU_APP_SECRET and FEISHU_LABELING_CHAT_ID)
|
||||
|
||||
|
||||
def send_chat_text(text: str) -> dict[str, Any]:
|
||||
if not is_notify_configured():
|
||||
return {"ok": False, "message": "未配置 FEISHU_LABELING_CHAT_ID"}
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
token = _get_tenant_access_token(client)
|
||||
resp = client.post(
|
||||
IM_MSG_URL,
|
||||
params={"receive_id_type": "chat_id"},
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"receive_id": FEISHU_LABELING_CHAT_ID,
|
||||
"msg_type": "text",
|
||||
"content": json.dumps({"text": text}, ensure_ascii=False),
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
return {"ok": False, "message": data.get("msg") or "send failed"}
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
def send_chat_async(text: str) -> None:
|
||||
"""异步发送飞书消息，不阻塞主流程。"""
|
||||
import threading
|
||||
threading.Thread(target=_send_safe, args=(text,), daemon=True, name="feishu-notify").start()
|
||||
|
||||
|
||||
def _send_safe(text: str) -> None:
|
||||
try:
|
||||
send_chat_text(text)
|
||||
except Exception:
|
||||
pass # 通知失败不影响业务流程
|
||||
|
||||
|
||||
def notify_batch_progress(
|
||||
*,
|
||||
delivery_id: str,
|
||||
task: str,
|
||||
batch_name: str,
|
||||
progress: str,
|
||||
link: str,
|
||||
) -> dict[str, Any]:
|
||||
text = (
|
||||
f"[HSAP] 标注进度 {delivery_id or batch_name}\n"
|
||||
f"任务: {task} / 批次: {batch_name}\n"
|
||||
f"进度: {progress}\n"
|
||||
f"链接: {link}"
|
||||
)
|
||||
return send_chat_text(text)
|
||||
Reference in New Issue
Block a user