feat: initial HSAP platform

Huaxu Sentinel Active Safety Platform with embedded algorithm code,
Docker Compose setup, and vendored dataset scaffolds for clone-and-run.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-05-25 16:59:59 +08:00
commit 7c43b44c57
1619 changed files with 373355 additions and 0 deletions

View File

@@ -0,0 +1,4 @@
"""华胥智能主动安全算法迭代平台。"""
from as_platform import sdk # noqa: F401
__all__ = ["sdk"]

View File

View File

@@ -0,0 +1,30 @@
"""ingest_flow感知 returned 批次 → 提交 build 审核。"""
from __future__ import annotations
from typing import Any
from as_platform.agents.tools import invoke_tool, submit_build_for_batch
from as_platform.agents.trace import start_trace, trace_span
def run_ingest_flow(*, task: str = "dam", submitted_by: str = "agent") -> dict[str, Any]:
trace_id = start_trace("ingest_flow", task=task)
with trace_span("list_pending"):
report = invoke_tool("list_pending_batches")
submitted = []
for batch in report.get("batches", []):
if batch.get("task") != task:
continue
if batch.get("stage") != "returned":
continue
with trace_span("submit_build", batch=batch.get("batch")):
apr = submit_build_for_batch(
task=task,
batch=batch["batch"],
pack=batch.get("pack") or "dms_v2",
submitted_by=submitted_by,
)
submitted.append(apr)
return {"trace_id": trace_id, "submitted": submitted, "count": len(submitted)}

View File

@@ -0,0 +1,20 @@
"""labeling_flow列出 raw_pool / out_for_labeling 批次。"""
from __future__ import annotations
from typing import Any
from as_platform.agents.tools import invoke_tool
from as_platform.agents.trace import start_trace, trace_span
def run_labeling_flow(*, task: str | None = None) -> dict[str, Any]:
trace_id = start_trace("labeling_flow", task=task)
with trace_span("list_pending"):
report = invoke_tool("list_pending_batches")
batches = [
b for b in report.get("batches", [])
if b.get("stage") in ("raw_pool", "out_for_labeling", "returned")
and (task is None or b.get("task") == task)
]
return {"trace_id": trace_id, "batches": batches, "count": len(batches)}

View File

@@ -0,0 +1,18 @@
"""train_promote_flow提交 train 审核platform 轨)。"""
from __future__ import annotations
from typing import Any
from as_platform.agents.tools import get_model_versions, invoke_tool, submit_train_job
from as_platform.agents.trace import start_trace, trace_span
def run_train_promote_flow(*, task: str = "dam", submitted_by: str = "agent") -> dict[str, Any]:
trace_id = start_trace("train_promote_flow", task=task)
with trace_span("get_versions"):
versions = get_model_versions(task)
with trace_span("submit_train"):
apr = submit_train_job("dms", task, track="platform", submitted_by=submitted_by)
return {"trace_id": trace_id, "versions_before": versions, "approval": apr}

View File

@@ -0,0 +1,86 @@
"""LangChain 风格 Tool 注册(纯 Python + 可选 langchain"""
from __future__ import annotations
from typing import Any, Callable
from as_platform.audit.queue import submit_approval
from as_platform.data.core import get_catalog, get_pending_report, load_wf
from as_platform.jobs.queue import get_job, list_jobs
import yaml
from pathlib import Path
WORKSPACE = Path(__file__).resolve().parents[2]
def list_pending_batches() -> dict[str, Any]:
return get_pending_report()
def get_dataset_catalog() -> dict[str, Any]:
return get_catalog()
def submit_build_for_batch(task: str, batch: str, pack: str = "dms_v2", submitted_by: str | None = None) -> dict:
return submit_approval(
"build_dms",
{"task": task, "pack": pack, "batch": batch},
submitted_by=submitted_by,
note=f"agent build {batch}",
)
def submit_train_job(project: str, task: str, track: str = "platform", submitted_by: str | None = None) -> dict:
action = "train_dms" if project == "dms" else "train_lane"
params: dict[str, Any] = {"track": track}
if project == "dms":
params["task"] = task
return submit_approval(action, params, submitted_by=submitted_by, note=f"agent train {project}/{task}")
def get_job_status(job_id: str) -> dict[str, Any] | None:
return get_job(job_id)
def get_model_versions(task: str) -> dict[str, Any]:
root = WORKSPACE / "datasets/dms/manifests/train_versions.yaml"
if not root.is_file():
return {}
data = yaml.safe_load(root.read_text(encoding="utf-8"))
return data.get(task, {})
TOOL_REGISTRY: dict[str, Callable[..., Any]] = {
"list_pending_batches": list_pending_batches,
"get_dataset_catalog": get_dataset_catalog,
"submit_build_for_batch": submit_build_for_batch,
"submit_train_job": submit_train_job,
"get_job_status": get_job_status,
"get_model_versions": get_model_versions,
}
def invoke_tool(name: str, **kwargs: Any) -> Any:
fn = TOOL_REGISTRY.get(name)
if not fn:
raise ValueError(f"未知 tool: {name}")
return fn(**kwargs)
def as_langchain_tools() -> list[Any]:
try:
from langchain_core.tools import tool
except ImportError:
return []
@tool
def t_list_pending_batches() -> dict:
"""列出待处理批次与送标状态。"""
return list_pending_batches()
@tool
def t_get_dataset_catalog() -> dict:
"""获取 DMS/Lane 数据目录统计。"""
return get_dataset_catalog()
return [t_list_pending_batches, t_get_dataset_catalog]

View File

@@ -0,0 +1,62 @@
"""LangSmith 式 tracemanifests/trace_log.jsonl"""
from __future__ import annotations
import json
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from typing import Any, Iterator
from as_platform.config import TRACE_LOG, MANIFESTS
_current_trace: str | None = None
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def start_trace(name: str, **meta: Any) -> str:
global _current_trace
trace_id = f"trace-{uuid.uuid4().hex[:12]}"
_current_trace = trace_id
_append({"type": "trace_start", "trace_id": trace_id, "name": name, "ts": _now(), **meta})
return trace_id
def _append(entry: dict[str, Any]) -> None:
MANIFESTS.mkdir(parents=True, exist_ok=True)
if not TRACE_LOG.is_file():
TRACE_LOG.write_text("", encoding="utf-8")
with TRACE_LOG.open("a", encoding="utf-8") as f:
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
@contextmanager
def trace_span(span_type: str, **fields: Any) -> Iterator[None]:
span_id = f"span-{uuid.uuid4().hex[:8]}"
_append({"type": span_type, "span_id": span_id, "trace_id": _current_trace, "ts": _now(), **fields})
try:
yield
finally:
_append({"type": f"{span_type}_end", "span_id": span_id, "trace_id": _current_trace, "ts": _now()})
def get_trace(trace_id: str) -> list[dict[str, Any]]:
if not TRACE_LOG.is_file():
return []
return [json.loads(l) for l in TRACE_LOG.read_text().strip().splitlines() if l.strip() and trace_id in l]
def list_traces(limit: int = 50) -> list[str]:
if not TRACE_LOG.is_file():
return []
ids = []
for line in TRACE_LOG.read_text().strip().splitlines():
try:
o = json.loads(line)
if o.get("type") == "trace_start" and o.get("trace_id"):
ids.append(o["trace_id"])
except json.JSONDecodeError:
pass
return list(reversed(ids[-limit:]))

View File

@@ -0,0 +1,4 @@
"""API 入口。"""
from as_platform.api.server import app, main
__all__ = ["app", "main"]

View File

@@ -0,0 +1,14 @@
import sys
from pathlib import Path
_ROOT = Path(__file__).resolve().parents[3] # HSAP repo root
_PLATFORM = _ROOT / "platform"
for p in (_ROOT, _PLATFORM):
s = str(p)
if s not in sys.path:
sys.path.insert(0, s)
from as_platform.api.server import main
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,98 @@
"""认证与用户管理 API。"""
from __future__ import annotations
from typing import Annotated, Any
from urllib.parse import urlencode
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import RedirectResponse
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from as_platform.auth.deps import get_current_user, require_permission
from as_platform.auth.feishu import build_authorize_url, exchange_code, is_feishu_configured, verify_state
from as_platform.auth.jwt import create_access_token
from as_platform.auth.users import get_or_create_dev_user, list_users, set_user_roles, upsert_feishu_user
from as_platform.config import DEV_AUTH_ENABLED, FRONTEND_URL
from as_platform.db.engine import get_db
from as_platform.db.init_db import user_to_dict
from as_platform.db.models import User
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
class DevLoginBody(BaseModel):
name: str = "开发用户"
class SetRolesBody(BaseModel):
role_codes: list[str] = Field(default_factory=list)
@router.get("/config")
def auth_config() -> dict[str, Any]:
return {
"feishu_enabled": is_feishu_configured(),
"dev_auth_enabled": DEV_AUTH_ENABLED and not is_feishu_configured(),
}
@router.get("/feishu/authorize")
def feishu_authorize():
if not is_feishu_configured():
raise HTTPException(503, "未配置飞书应用,请设置 FEISHU_APP_ID / FEISHU_APP_SECRET")
url, _ = build_authorize_url()
return RedirectResponse(url)
@router.get("/feishu/callback")
def feishu_callback(code: str, state: str, db: Annotated[Session, Depends(get_db)]):
if not verify_state(state):
raise HTTPException(400, "无效的 state请重新登录")
try:
info = exchange_code(code)
except Exception as e:
raise HTTPException(502, f"飞书登录失败: {e}") from e
user = upsert_feishu_user(db, info)
db.commit()
token = create_access_token(user.id)
# 避免 BrowserRouter 直达路径 404统一回根路径并附带 token
qs = urlencode({"token": token})
return RedirectResponse(f"{FRONTEND_URL}/?{qs}")
@router.post("/dev/login")
def dev_login(body: DevLoginBody, db: Annotated[Session, Depends(get_db)]):
if not DEV_AUTH_ENABLED or is_feishu_configured():
raise HTTPException(403, "开发登录未启用")
user = get_or_create_dev_user(db, body.name)
db.commit()
token = create_access_token(user.id)
return {"access_token": token, "user": user_to_dict(user)}
@router.get("/me")
def auth_me(user: Annotated[User, Depends(get_current_user)]):
return user_to_dict(user)
@router.get("/users")
def auth_list_users(
_user: Annotated[User, Depends(require_permission("admin:users"))],
db: Annotated[Session, Depends(get_db)],
):
return {"items": [user_to_dict(u) for u in list_users(db)]}
@router.put("/users/{user_id}/roles")
def auth_set_roles(
user_id: int,
body: SetRolesBody,
_user: Annotated[User, Depends(require_permission("admin:users"))],
db: Annotated[Session, Depends(get_db)],
):
user = set_user_roles(db, user_id, body.role_codes)
if not user:
raise HTTPException(404, "用户不存在")
db.commit()
return user_to_dict(user)

View File

@@ -0,0 +1,584 @@
#!/usr/bin/env python3
"""统一 API + React Web。cd HSAP && PYTHONPATH=platform python -m as_platform.api.server"""
from __future__ import annotations
import argparse
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Annotated, Any
import threading
try:
from fastapi import Depends, FastAPI, File, Form, HTTPException, Query, UploadFile
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, Response
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, Field
except ImportError as e:
raise SystemExit("需要安装: pip install fastapi uvicorn pydantic sqlalchemy python-jose httpx") from e
from as_platform.agents.graphs.ingest_flow import run_ingest_flow
from as_platform.agents.graphs.labeling_flow import run_labeling_flow
from as_platform.agents.graphs.train_promote_flow import run_train_promote_flow
from as_platform.agents.tools import TOOL_REGISTRY, invoke_tool
from as_platform.agents.trace import get_trace, list_traces
from as_platform.api.auth_routes import router as auth_router
from as_platform.audit.queue import (
ACTION_LABELS,
ACTIONS_REQUIRING_APPROVAL,
approve_and_execute,
get_approval,
list_approvals,
reject_approval,
submit_approval,
)
from as_platform.audit.preview import find_image_ref, list_scope_images, render_overlay, resolve_approval_scope
from as_platform.auth.deps import can_submit_action, get_current_user, require_any_permission, require_permission
from as_platform.config import IS_POSTGRES, PLATFORM_DIR, PLATFORM_WEB, WORKSPACE
from as_platform.data.core import get_catalog, get_pending_report, register_batch, warmup_catalog_cache
from as_platform.data.ingest import UnknownFormatError, inspect_uploaded_dataset
from as_platform.data.lake import (
create_uploaded_candidate,
get_candidate,
link_candidate_analysis_job,
list_candidates as list_data_candidates,
write_candidate_upload,
)
from as_platform.data.organize import organize_batch
from as_platform.db.engine import check_connection
from as_platform.db.init_db import init_database
from as_platform.db.models import User
from as_platform.jobs.queue import enqueue_job, get_job, list_jobs
from as_platform.redis.bus import ping_redis
from as_platform.training.service import (
TRAINING_ACTIONS,
create_training_submission,
get_model_registry,
get_training_record,
list_training_records,
)
@asynccontextmanager
async def lifespan(_app: FastAPI):
init_database()
threading.Thread(target=warmup_catalog_cache, daemon=True, name="catalog-warmup").start()
yield
app = FastAPI(title="华胥智能主动安全平台", version="1.1.0", lifespan=lifespan)
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
app.include_router(auth_router)
_UI_DIR: Path | None = None
class SubmitApprovalBody(BaseModel):
action: str
params: dict[str, Any] = Field(default_factory=dict)
submitted_by: str | None = None
note: str | None = None
class ReviewBody(BaseModel):
reviewed_by: str | None = None
comment: str | None = None
class RegisterBatchBody(BaseModel):
project: str
task: str | None = None
batch: str
pack: str | None = None
stage: str = "returned"
engineer: str | None = None
location: str = "inbox"
submitted_by: str | None = None
skip_audit: bool = False
class BuildFromBatchBody(BaseModel):
project: str = "dms"
task: str
batch: str
pack: str = "dms_v2"
location: str = "inbox"
submitted_by: str | None = None
note: str | None = None
class OrganizeBody(BaseModel):
batch_path: str
task: str | None = None
class AgentInvokeBody(BaseModel):
graph: str = "ingest_flow"
params: dict[str, Any] = Field(default_factory=dict)
class ToolInvokeBody(BaseModel):
tool: str
params: dict[str, Any] = Field(default_factory=dict)
class CreateTrainingBody(BaseModel):
action: str
params: dict[str, Any] = Field(default_factory=dict)
note: str | None = None
class InspectUploadBody(BaseModel):
project: str
task: str | None = None
source_path: str
@app.get("/api/v1/health")
def health() -> dict[str, str]:
db_ok = check_connection()
redis_ok = ping_redis()
ok = db_ok and redis_ok
return {
"status": "ok" if ok else "degraded",
"workspace": str(WORKSPACE),
"database": "postgresql" if IS_POSTGRES else "sqlite",
"db_connected": str(db_ok).lower(),
"redis_connected": str(redis_ok).lower(),
}
@app.get("/api/v1/pending")
def api_pending(_user: Annotated[User, Depends(require_permission("read:pending"))]) -> dict[str, Any]:
return get_pending_report()
@app.get("/api/v1/catalog")
def api_catalog(
_user: Annotated[User, Depends(require_permission("read:catalog"))],
refresh: bool = Query(False),
) -> dict[str, Any]:
return get_catalog(refresh=refresh)
@app.get("/api/v1/catalog/dms/{task}")
def api_catalog_dms(
task: str,
_user: Annotated[User, Depends(require_permission("read:catalog"))],
refresh: bool = Query(False),
) -> dict[str, Any]:
full = get_catalog(refresh=refresh)
if task not in (full.get("dms") or {}):
raise HTTPException(404, f"未知 DMS 任务: {task}")
return {"task": task, **full["dms"][task]}
@app.get("/api/v1/catalog/lane/{pack}")
def api_catalog_lane(
pack: str,
_user: Annotated[User, Depends(require_permission("read:catalog"))],
refresh: bool = Query(False),
) -> dict[str, Any]:
full = get_catalog(refresh=refresh)
if pack not in (full.get("lane") or {}):
raise HTTPException(404, f"未知 Lane 包: {pack}")
return {"pack": pack, **full["lane"][pack]}
@app.get("/api/v1/actions")
def api_actions(_user: Annotated[User, Depends(require_permission("read:audit"))]) -> dict[str, Any]:
return {"actions": [{"id": k, "label": ACTION_LABELS.get(k, k)} for k in sorted(ACTIONS_REQUIRING_APPROVAL)]}
@app.get("/api/v1/jobs")
def api_jobs(
_user: Annotated[User, Depends(require_permission("read:jobs"))],
status: str | None = None,
limit: int = Query(100, le=500),
) -> dict[str, Any]:
return {"items": list_jobs(status=status, limit=limit)}
@app.get("/api/v1/jobs/{job_id}")
def api_job(job_id: str, _user: Annotated[User, Depends(require_permission("read:jobs"))]) -> dict[str, Any]:
job = get_job(job_id)
if not job:
raise HTTPException(404, "Job 不存在")
return job
@app.get("/api/v1/training/actions")
def api_training_actions(_user: Annotated[User, Depends(require_permission("read:jobs"))]) -> dict[str, Any]:
return {
"actions": [
{"id": action, "label": ACTION_LABELS.get(action, action)}
for action in sorted(TRAINING_ACTIONS)
]
}
@app.get("/api/v1/training/records")
def api_training_records(
_user: Annotated[User, Depends(require_permission("read:jobs"))],
project: str | None = None,
kind: str | None = None,
status: str | None = None,
task: str | None = None,
limit: int = Query(100, ge=1, le=500),
) -> dict[str, Any]:
return list_training_records(project=project, kind=kind, status=status, task=task, limit=limit)
@app.get("/api/v1/training/records/{job_id}")
def api_training_record(
job_id: str,
_user: Annotated[User, Depends(require_permission("read:jobs"))],
) -> dict[str, Any]:
rec = get_training_record(job_id)
if not rec:
raise HTTPException(404, "训练记录不存在")
return rec
@app.get("/api/v1/training/models")
def api_training_models(
_user: Annotated[User, Depends(require_permission("read:jobs"))],
project: str = Query("dms"),
task: str | None = None,
) -> dict[str, Any]:
return get_model_registry(project=project, task=task)
@app.post("/api/v1/training/records")
def api_create_training(
body: CreateTrainingBody,
user: Annotated[User, Depends(get_current_user)],
) -> dict[str, Any]:
if not can_submit_action(user, body.action):
raise HTTPException(403, f"无权提交: {body.action}")
try:
return create_training_submission(
body.action,
body.params,
submitted_by=user.name,
submitted_by_user_id=user.id,
note=body.note,
)
except ValueError as e:
raise HTTPException(400, str(e)) from e
@app.get("/api/v1/traces")
def api_traces(_user: Annotated[User, Depends(get_current_user)], limit: int = 50) -> dict[str, Any]:
return {"trace_ids": list_traces(limit=limit)}
@app.get("/api/v1/traces/{trace_id}")
def api_trace(trace_id: str, _user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
spans = get_trace(trace_id)
if not spans:
raise HTTPException(404, "Trace 不存在")
return {"trace_id": trace_id, "spans": spans}
@app.get("/api/v1/agents/tools")
def api_tools(_user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
return {"tools": list(TOOL_REGISTRY.keys())}
@app.post("/api/v1/agents/tools/invoke")
def api_tool_invoke(body: ToolInvokeBody, _user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
try:
return {"result": invoke_tool(body.tool, **body.params)}
except ValueError as e:
raise HTTPException(400, str(e)) from e
@app.post("/api/v1/agents/invoke")
def api_agent_invoke(body: AgentInvokeBody, _user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
graphs = {
"ingest_flow": run_ingest_flow,
"labeling_flow": run_labeling_flow,
"train_promote_flow": run_train_promote_flow,
}
fn = graphs.get(body.graph)
if not fn:
raise HTTPException(400, f"未知 graph: {body.graph}")
return fn(**body.params)
@app.get("/api/v1/approvals")
def api_list_approvals(
_user: Annotated[User, Depends(require_permission("read:audit"))],
status: str | None = None,
limit: int = Query(100, le=500),
) -> dict[str, Any]:
return {"items": list_approvals(status=status, limit=limit)}
@app.get("/api/v1/approvals/{record_id}")
def api_get_approval(record_id: str, _user: Annotated[User, Depends(require_permission("read:audit"))]) -> dict[str, Any]:
rec = get_approval(record_id)
if not rec:
raise HTTPException(404, "审核单不存在")
return rec
@app.get("/api/v1/approvals/{record_id}/preview")
def api_approval_preview(
record_id: str,
_user: Annotated[User, Depends(require_permission("read:audit"))],
) -> dict[str, Any]:
rec = get_approval(record_id)
if not rec:
raise HTTPException(404, "审核单不存在")
try:
scope = resolve_approval_scope(rec["action"], rec.get("params") or {})
except ValueError as e:
raise HTTPException(400, str(e)) from e
batch_summaries = []
for b in scope.get("batches") or []:
batch_dir = Path(b["path"])
batch_summaries.append(
{
"batch": b.get("batch"),
"location": b.get("location"),
"path": str(batch_dir) if batch_dir.is_dir() else None,
"exists": batch_dir.is_dir(),
}
)
return {
"approval": rec,
"scope_label": scope.get("scope_label"),
"task": scope.get("task"),
"pack": scope.get("pack"),
"class_names": scope.get("class_names"),
"batches": batch_summaries,
}
@app.get("/api/v1/approvals/{record_id}/images")
def api_approval_images(
record_id: str,
_user: Annotated[User, Depends(require_permission("read:audit"))],
offset: int = Query(0, ge=0),
limit: int = Query(60, ge=1, le=200),
) -> dict[str, Any]:
rec = get_approval(record_id)
if not rec:
raise HTTPException(404, "审核单不存在")
try:
scope = resolve_approval_scope(rec["action"], rec.get("params") or {})
return list_scope_images(scope, offset=offset, limit=limit)
except ValueError as e:
raise HTTPException(400, str(e)) from e
@app.get("/api/v1/approvals/{record_id}/images/{image_id}")
def api_approval_image(
record_id: str,
image_id: str,
_user: Annotated[User, Depends(require_permission("read:audit"))],
thumb: bool = Query(True),
) -> Response:
rec = get_approval(record_id)
if not rec:
raise HTTPException(404, "审核单不存在")
try:
scope = resolve_approval_scope(rec["action"], rec.get("params") or {})
ref = find_image_ref(scope, image_id)
if not ref or not ref.image_path.is_file():
raise HTTPException(404, "图像不存在")
class_names = scope.get("class_names") or {}
max_size = 480 if thumb else 1920
data = render_overlay(ref.image_path, ref.label_path, class_names, max_size=max_size)
return Response(content=data, media_type="image/jpeg")
except ValueError as e:
raise HTTPException(400, str(e)) from e
@app.post("/api/v1/approvals/submit")
def api_submit(body: SubmitApprovalBody, user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
if not can_submit_action(user, body.action):
raise HTTPException(403, f"无权提交: {body.action}")
try:
return submit_approval(
body.action, body.params,
submitted_by=user.name,
submitted_by_user_id=user.id,
note=body.note,
)
except ValueError as e:
raise HTTPException(400, str(e)) from e
@app.post("/api/v1/approvals/submit-build-batch")
def api_submit_build_batch(body: BuildFromBatchBody, user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
if not can_submit_action(user, "build_dms"):
raise HTTPException(403, "无权提交 build")
params: dict[str, Any] = {"task": body.task, "pack": body.pack}
if body.location == "inbox":
params["batch"] = body.batch
else:
params["all_sources"] = True
return submit_approval(
"build_dms", params,
submitted_by=user.name,
submitted_by_user_id=user.id,
note=body.note or f"入库 {body.batch}",
)
@app.post("/api/v1/approvals/{record_id}/approve")
def api_approve(record_id: str, body: ReviewBody, user: Annotated[User, Depends(require_permission("write:approval_review"))]) -> dict[str, Any]:
try:
return approve_and_execute(
record_id,
reviewed_by=user.name,
reviewed_by_user_id=user.id,
comment=body.comment,
)
except ValueError as e:
raise HTTPException(400, str(e)) from e
@app.post("/api/v1/approvals/{record_id}/reject")
def api_reject(record_id: str, body: ReviewBody, user: Annotated[User, Depends(require_permission("write:approval_review"))]) -> dict[str, Any]:
try:
return reject_approval(
record_id,
reviewed_by=user.name,
reviewed_by_user_id=user.id,
comment=body.comment,
)
except ValueError as e:
raise HTTPException(400, str(e)) from e
@app.post("/api/v1/register-batch")
def api_register_batch(body: RegisterBatchBody, user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
if body.skip_audit:
if not can_submit_action(user, "register_batch"):
raise HTTPException(403, "无权登记批次")
try:
return register_batch(None, body.project, body.task, body.batch, pack=body.pack, stage=body.stage, engineer=body.engineer, location=body.location)
except (ValueError, FileNotFoundError) as e:
raise HTTPException(400, str(e)) from e
if not can_submit_action(user, "register_batch"):
raise HTTPException(403, "无权提交登记审核")
return submit_approval(
"register_batch",
body.model_dump(exclude={"submitted_by", "skip_audit"}),
submitted_by=user.name,
submitted_by_user_id=user.id,
note="登记 batch.meta",
)
@app.post("/api/v1/data/organize")
def api_organize(body: OrganizeBody, _user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
try:
return organize_batch(Path(body.batch_path), task=body.task)
except FileNotFoundError as e:
raise HTTPException(404, str(e)) from e
@app.post("/api/v1/data/inspect-upload")
def api_inspect_upload(body: InspectUploadBody, _user: Annotated[User, Depends(require_permission("read:catalog"))]) -> dict[str, Any]:
try:
result = inspect_uploaded_dataset(body.project, body.task, body.source_path)
return {"ok": True, "normalized": result.to_dict()}
except FileNotFoundError as e:
raise HTTPException(404, str(e)) from e
except UnknownFormatError as e:
raise HTTPException(400, str(e)) from e
@app.post("/api/v1/data/upload/file")
async def api_upload_file(
project: Annotated[str, Form()],
user: Annotated[User, Depends(require_any_permission("write:approval_submit", "write:approval_submit:register"))],
task: Annotated[str | None, Form()] = None,
file: UploadFile = File(...),
) -> dict[str, Any]:
if not file.filename:
raise HTTPException(400, "上传文件名不能为空")
try:
candidate = create_uploaded_candidate(
project=project,
task=task,
original_name=file.filename,
upload_size_bytes=0,
submitted_by_name=user.name if user else None,
submitted_by_user_id=user.id if user else None,
)
write_candidate_upload(candidate["id"], file.file)
job = enqueue_job("analyze_uploaded_dataset", {"candidate_id": candidate["id"]}, async_run=True)
link_candidate_analysis_job(candidate["id"], job["id"])
updated = get_candidate(candidate["id"]) or candidate
return {"ok": True, "candidate": updated, "job": job}
except ValueError as e:
raise HTTPException(400, str(e)) from e
finally:
await file.close()
@app.get("/api/v1/data/candidates")
def api_data_candidates(
_user: Annotated[User, Depends(require_permission("read:catalog"))],
limit: int = Query(50, ge=1, le=500),
) -> dict[str, Any]:
return {"items": list_data_candidates(limit=limit)}
@app.get("/api/v1/data/candidates/{candidate_id}")
def api_data_candidate(candidate_id: str, _user: Annotated[User, Depends(require_permission("read:catalog"))]) -> dict[str, Any]:
item = get_candidate(candidate_id)
if not item:
raise HTTPException(404, "candidate 不存在")
return item
def _mount_ui() -> None:
global _UI_DIR
for ui in (PLATFORM_WEB, PLATFORM_DIR / "web" / "dist"):
if (ui / "index.html").is_file():
_UI_DIR = ui
app.mount("/assets", StaticFiles(directory=str(ui / "assets")), name="ui-assets")
return
_mount_ui()
@app.get("/{full_path:path}", include_in_schema=False)
def spa_fallback(full_path: str):
if full_path.startswith("api/"):
raise HTTPException(404, "Not Found")
if not _UI_DIR:
raise HTTPException(404, "UI not built")
safe = Path(full_path).as_posix().lstrip("/")
target = (_UI_DIR / safe).resolve()
ui_root = _UI_DIR.resolve()
if safe and target.is_file() and target.is_relative_to(ui_root):
return FileResponse(target)
return FileResponse(_UI_DIR / "index.html")
@app.head("/{full_path:path}", include_in_schema=False)
def spa_fallback_head(full_path: str):
return spa_fallback(full_path)
def main() -> None:
import uvicorn
ap = argparse.ArgumentParser(description="华胥智能主动安全平台")
ap.add_argument("--host", default="127.0.0.1")
ap.add_argument("--port", type=int, default=8787)
args = ap.parse_args()
uvicorn.run(app, host=args.host, port=args.port)
if __name__ == "__main__":
main()

View File

View File

@@ -0,0 +1,355 @@
"""审核单关联的送标/回传数据:解析范围、列举图像、渲染 GT 叠加。"""
from __future__ import annotations
import hashlib
import io
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Sequence
import yaml
from PIL import Image, ImageDraw, ImageFont
from as_platform.data.batch import IMG_EXTS
from as_platform.data.core import load_wf, proj_root, resolve_pack_dir
IMAGE_EXTS = tuple(ext.lower() for ext in IMG_EXTS) + tuple(ext.upper() for ext in IMG_EXTS if ext.islower())
@dataclass(frozen=True)
class ImageRef:
image_path: Path
label_path: Path | None
batch: str
location: str
split: str
@property
def id(self) -> str:
key = f"{self.image_path}|{self.label_path or ''}"
return hashlib.sha256(key.encode()).hexdigest()[:16]
def _find_image(images_dir: Path, stem: str) -> Path | None:
for ext in IMAGE_EXTS:
p = images_dir / f"{stem}{ext}"
if p.is_file():
return p
return None
def _parse_yolo_line(line: str) -> dict[str, Any] | None:
parts = line.strip().split()
if len(parts) < 5:
return None
try:
class_id = int(float(parts[0]))
cx, cy, w, h = map(float, parts[1:5])
except Exception:
return None
keypoints: list[tuple[float, float, float]] = []
rest = parts[5:]
if len(rest) >= 3:
n = len(rest) // 3
for i in range(n):
keypoints.append((float(rest[i * 3]), float(rest[i * 3 + 1]), float(rest[i * 3 + 2])))
return {"class_id": class_id, "bbox": (cx, cy, w, h), "keypoints": keypoints}
def parse_label_file(label_path: Path) -> list[dict[str, Any]]:
if not label_path.is_file():
return []
out: list[dict[str, Any]] = []
for raw in label_path.read_text(encoding="utf-8", errors="ignore").splitlines():
parsed = _parse_yolo_line(raw)
if parsed is not None:
out.append(parsed)
return out
def _load_font(size: int) -> ImageFont.ImageFont:
for p in (
"/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
"/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc",
"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
):
if Path(p).exists():
try:
return ImageFont.truetype(p, size)
except Exception:
continue
return ImageFont.load_default()
def _yolo_bbox_to_xyxy(bbox: tuple[float, float, float, float], width: int, height: int) -> tuple[int, int, int, int]:
cx, cy, bw, bh = bbox
x1 = int((cx - bw / 2.0) * width)
y1 = int((cy - bh / 2.0) * height)
x2 = int((cx + bw / 2.0) * width)
y2 = int((cy + bh / 2.0) * height)
return (
max(0, min(width - 1, x1)),
max(0, min(height - 1, y1)),
max(0, min(width - 1, x2)),
max(0, min(height - 1, y2)),
)
def render_overlay(
image_path: Path,
label_path: Path | None,
class_names: dict[int, str],
*,
max_size: int | None = None,
) -> bytes:
with Image.open(image_path) as im:
base = im.convert("RGB")
if max_size and max(base.size) > max_size:
base.thumbnail((max_size, max_size), Image.Resampling.LANCZOS)
over = base.copy()
draw = ImageDraw.Draw(over)
w, h = over.size
font = _load_font(max(12, min(18, w // 40)))
anns = parse_label_file(label_path) if label_path else []
palette = [
(220, 20, 60),
(30, 144, 255),
(50, 205, 50),
(255, 165, 0),
(186, 85, 211),
(0, 206, 209),
]
for ann in anns:
cid = ann["class_id"]
color = palette[cid % len(palette)]
x1, y1, x2, y2 = _yolo_bbox_to_xyxy(ann["bbox"], w, h)
draw.rectangle((x1, y1, x2, y2), outline=color, width=max(2, w // 320))
label = class_names.get(cid, f"class_{cid}")
draw.text((x1 + 2, max(0, y1 - 16)), label, fill=color, font=font)
for kx, ky, kv in ann.get("keypoints") or []:
if kv <= 0:
continue
px, py = int(kx * w), int(ky * h)
r = max(2, w // 400)
draw.ellipse((px - r, py - r, px + r, py + r), outline=color, fill=color)
buf = io.BytesIO()
over.save(buf, format="JPEG", quality=88)
return buf.getvalue()
def _collect_from_split(batch_dir: Path, split: str, *, batch: str, location: str) -> list[ImageRef]:
if split:
images_dir = batch_dir / "images" / split
labels_dir = batch_dir / "labels" / split
else:
images_dir = batch_dir / "images"
labels_dir = batch_dir / "labels"
out: list[ImageRef] = []
if not images_dir.is_dir():
return out
label_stems: dict[str, Path] = {}
if labels_dir.is_dir():
for lp in labels_dir.glob("*.txt"):
label_stems[lp.stem] = lp
seen: set[str] = set()
for p in sorted(images_dir.iterdir()):
if not p.is_file() or p.suffix.lower() not in {e.lower() for e in IMG_EXTS}:
continue
stem = p.stem
seen.add(stem)
out.append(
ImageRef(
image_path=p.resolve(),
label_path=label_stems.get(stem),
batch=batch,
location=location,
split=split or "root",
)
)
for stem, lp in sorted(label_stems.items()):
if stem in seen:
continue
img = _find_image(images_dir, stem)
if img:
out.append(
ImageRef(
image_path=img.resolve(),
label_path=lp.resolve(),
batch=batch,
location=location,
split=split or "root",
)
)
return out
def collect_batch_images(batch_dir: Path, *, batch: str, location: str) -> list[ImageRef]:
if not batch_dir.is_dir():
return []
refs: list[ImageRef] = []
for split in ("train", "val", "test", ""):
refs.extend(_collect_from_split(batch_dir, split, batch=batch, location=location))
# 去重flat 与 train 可能重叠)
dedup: dict[str, ImageRef] = {}
for ref in refs:
dedup[str(ref.image_path)] = ref
return sorted(dedup.values(), key=lambda r: (r.batch, r.split, r.image_path.name))
def _dms_task_cfg(root: Path, wf: dict, task: str) -> tuple[dict, dict]:
reg_path = root / wf["projects"]["dms"]["registry"]
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
if task not in reg.get("tasks", {}):
raise ValueError(f"未知 task: {task}")
return reg, reg["tasks"][task]
def resolve_approval_scope(action: str, params: dict[str, Any]) -> dict[str, Any]:
"""解析审核单对应的数据目录与类别名。"""
p = params or {}
wf = load_wf()
if action in ("build_dms", "register_batch"):
task = p.get("task")
if not task:
raise ValueError("缺少 task 参数")
root = proj_root(wf, "dms")
reg, tcfg = _dms_task_cfg(root, wf, task)
src_sub = (reg.get("ingest") or {}).get("sources_subdir", "sources")
pack = p.get("pack") or "dms_v2"
batches: list[dict[str, Any]] = []
location = p.get("location", "inbox")
if action == "build_dms" and p.get("all_sources"):
location = "sources"
if action == "build_dms" and p.get("batch") and not p.get("all_sources"):
location = "inbox"
if location == "inbox":
batch_name = p.get("batch")
if batch_name:
batches.append({"path": root / "inbox" / task / batch_name, "batch": batch_name, "location": "inbox"})
else:
ib = root / "inbox" / task
if ib.is_dir():
for d in sorted(ib.iterdir()):
if d.is_dir() and not d.name.startswith("."):
batches.append({"path": d, "batch": d.name, "location": "inbox"})
else:
pack_dir = resolve_pack_dir("dms", root, wf, pack)
src_root = pack_dir / tcfg.get("task_dir", task) / src_sub
batch_name = p.get("batch")
if batch_name:
batches.append({"path": src_root / batch_name, "batch": batch_name, "location": "sources"})
elif src_root.is_dir():
for d in sorted(src_root.iterdir()):
if d.is_dir() and d.name not in ("_ingested", "_merged") and not d.name.startswith("."):
batches.append({"path": d, "batch": d.name, "location": "sources"})
names = tcfg.get("names") or {}
class_names = {int(k): v for k, v in names.items()} if isinstance(names, dict) else {i: n for i, n in enumerate(names)}
return {
"project": "dms",
"task": task,
"pack": pack,
"scope_label": f"DMS · {task} · {pack}" + (" · 全部 sources" if location == "sources" and not p.get("batch") else ""),
"class_names": class_names,
"batches": batches,
}
if action in ("train_dms", "promote_dms", "eval_dms"):
task = p.get("task")
if not task:
raise ValueError("缺少 task 参数")
root = proj_root(wf, "dms")
_, tcfg = _dms_task_cfg(root, wf, task)
pack = p.get("pack") or "dms_v2"
pack_dir = resolve_pack_dir("dms", root, wf, pack)
task_dir = pack_dir / tcfg.get("task_dir", task)
batches = [{"path": task_dir, "batch": f"{pack}/{task}", "location": "pack"}]
names = tcfg.get("names") or {}
class_names = {int(k): v for k, v in names.items()} if isinstance(names, dict) else {i: n for i, n in enumerate(names)}
label = "模型晋级" if action == "promote_dms" else ("评估" if action == "eval_dms" else "训练")
return {
"project": "dms",
"task": task,
"pack": pack,
"scope_label": f"DMS · {task} · {pack} · pack 数据({label}",
"class_names": class_names,
"batches": batches,
}
raise ValueError(f"暂不支持预览的动作: {action}")
def list_scope_images(scope: dict[str, Any], *, offset: int = 0, limit: int = 60) -> dict[str, Any]:
all_refs: list[ImageRef] = []
for b in scope.get("batches") or []:
batch_dir = Path(b["path"])
all_refs.extend(
collect_batch_images(batch_dir, batch=b.get("batch", batch_dir.name), location=b.get("location", ""))
)
dedup: dict[str, ImageRef] = {str(r.image_path): r for r in all_refs}
ordered = sorted(dedup.values(), key=lambda r: (r.batch, r.split, r.image_path.name))
total = len(ordered)
page = ordered[offset : offset + limit]
items = []
for ref in page:
anns = parse_label_file(ref.label_path) if ref.label_path else []
items.append(
{
"id": ref.id,
"batch": ref.batch,
"location": ref.location,
"split": ref.split,
"filename": ref.image_path.name,
"has_label": ref.label_path is not None and ref.label_path.is_file(),
"box_count": len(anns),
"missing_label": ref.label_path is None or not ref.label_path.is_file(),
}
)
return {"total": total, "offset": offset, "limit": limit, "items": items}
def find_image_ref(scope: dict[str, Any], image_id: str) -> ImageRef | None:
"""线性查找;审核场景批次有限,可接受。"""
batches = scope.get("batches") or []
for b in batches:
batch_dir = Path(b["path"])
refs = collect_batch_images(batch_dir, batch=b.get("batch", batch_dir.name), location=b.get("location", ""))
for ref in refs:
if ref.id == image_id:
return ref
return None
def image_to_item(ref: ImageRef) -> dict[str, Any]:
anns = parse_label_file(ref.label_path) if ref.label_path else []
return {
"id": ref.id,
"batch": ref.batch,
"location": ref.location,
"split": ref.split,
"filename": ref.image_path.name,
"has_label": ref.label_path is not None and ref.label_path.is_file(),
"box_count": len(anns),
"missing_label": ref.label_path is None or not ref.label_path.is_file(),
"annotations": [
{
"class_id": a["class_id"],
"class_name": None,
"bbox": a["bbox"],
"keypoints": a.get("keypoints") or [],
}
for a in anns
],
}

View File

@@ -0,0 +1,162 @@
"""审核队列SQLite"""
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from typing import Any
from as_platform.db.engine import session_scope
from as_platform.db.models import Approval, User
from as_platform.config import LANE_DATA_VIZ_ENABLED
ACTIONS_REQUIRING_APPROVAL = {
"build_dms", "build_lane", "enable_pack", "disable_pack",
"train_dms", "train_lane", "eval_dms", "promote_dms",
"pipeline_dms", "register_batch", "eval_lane", "visualize_dms", "visualize_lane",
}
ACTION_LABELS = {
"build_dms": "DMS 入库 (build)",
"build_lane": "车道线合并列表 (build lane)",
"enable_pack": "启用训练数据包",
"disable_pack": "停用训练数据包",
"train_dms": "DMS 训练",
"train_lane": "车道线训练",
"eval_dms": "DMS 评估",
"eval_lane": "车道线评估",
"visualize_dms": "DMS 检测可视化",
"visualize_lane": "车道线可视化",
"promote_dms": "DMS 模型晋级",
"pipeline_dms": "DMS 半自动流水线",
"register_batch": "登记批次元数据",
}
def _now() -> datetime:
return datetime.now(timezone.utc)
def _new_id() -> str:
return f"apr-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
def submit_approval(
action: str,
params: dict[str, Any],
*,
submitted_by: str | None = None,
submitted_by_user_id: int | None = None,
note: str | None = None,
auto_execute: bool = False,
) -> dict[str, Any]:
if action not in ACTIONS_REQUIRING_APPROVAL:
raise ValueError(f"未知动作: {action},允许: {sorted(ACTIONS_REQUIRING_APPROVAL)}")
if action == "visualize_lane" and not LANE_DATA_VIZ_ENABLED:
raise ValueError("车道线数据可视化暂未开放")
from as_platform.agents.trace import trace_span
with session_scope() as db:
rec = Approval(
id=_new_id(),
status="pending",
action=action,
action_label=ACTION_LABELS.get(action, action),
note=note,
submitted_by_name=submitted_by,
submitted_by_user_id=submitted_by_user_id,
submitted_at=_now(),
)
rec.set_params(params)
db.add(rec)
db.flush()
out = rec.to_dict()
with trace_span("approval_submit", approval_id=out["id"], action=action):
pass
if auto_execute:
return approve_and_execute(out["id"], reviewed_by="system", comment="auto_execute")
return out
def list_approvals(status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
with session_scope() as db:
q = db.query(Approval).order_by(Approval.submitted_at.desc())
if status:
q = q.filter(Approval.status == status)
return [a.to_dict() for a in q.limit(limit).all()]
def get_approval(record_id: str) -> dict[str, Any] | None:
with session_scope() as db:
rec = db.get(Approval, record_id)
return rec.to_dict() if rec else None
def _update(record_id: str, **patch: Any) -> dict[str, Any] | None:
with session_scope() as db:
rec = db.get(Approval, record_id)
if not rec:
return None
for k, v in patch.items():
if k == "result" and isinstance(v, dict):
rec.set_result(v)
elif hasattr(rec, k):
setattr(rec, k, v)
db.flush()
return rec.to_dict()
def approve_and_execute(
record_id: str,
*,
reviewed_by: str | None = None,
reviewed_by_user_id: int | None = None,
comment: str | None = None,
) -> dict[str, Any]:
rec = get_approval(record_id)
if not rec:
raise ValueError(f"审核单不存在: {record_id}")
if rec.get("status") != "pending":
raise ValueError(f"当前状态不可审批: {rec.get('status')}")
from as_platform.agents.trace import trace_span
from as_platform.jobs.queue import enqueue_job
_update(
record_id,
status="approved",
reviewed_by_name=reviewed_by,
reviewed_by_user_id=reviewed_by_user_id,
reviewed_at=_now(),
review_comment=comment,
)
with trace_span("approval_approved", approval_id=record_id, action=rec["action"]):
job = enqueue_job(rec["action"], rec.get("params") or {}, approval_id=record_id, async_run=True)
_update(record_id, job_id=job.get("id"), status="running")
return get_approval(record_id) or {}
def reject_approval(
record_id: str,
*,
reviewed_by: str | None = None,
reviewed_by_user_id: int | None = None,
comment: str | None = None,
) -> dict[str, Any]:
rec = get_approval(record_id)
if not rec:
raise ValueError(f"审核单不存在: {record_id}")
if rec.get("status") != "pending":
raise ValueError(f"当前状态不可驳回: {rec.get('status')}")
return _update(
record_id,
status="rejected",
reviewed_by_name=reviewed_by,
reviewed_by_user_id=reviewed_by_user_id,
reviewed_at=_now(),
review_comment=comment,
) or {}

View File

View File

@@ -0,0 +1,71 @@
"""FastAPI 认证依赖。"""
from __future__ import annotations
from typing import Annotated
from fastapi import Depends, HTTPException, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from sqlalchemy.orm import Session, joinedload
from as_platform.auth.jwt import decode_access_token
from as_platform.config import DEV_AUTH_ENABLED
from as_platform.db.engine import get_db
from as_platform.db.init_db import user_has_permission
from as_platform.db.models import Role, User
_bearer = HTTPBearer(auto_error=False)
def _load_user(db: Session, user_id: int) -> User | None:
return (
db.query(User)
.options(joinedload(User.roles).joinedload(Role.permissions))
.filter(User.id == user_id, User.is_active.is_(True))
.first()
)
def get_current_user_optional(
creds: Annotated[HTTPAuthorizationCredentials | None, Depends(_bearer)],
db: Annotated[Session, Depends(get_db)],
) -> User | None:
if not creds:
return None
payload = decode_access_token(creds.credentials)
if not payload or "sub" not in payload:
return None
return _load_user(db, int(payload["sub"]))
def get_current_user(
user: Annotated[User | None, Depends(get_current_user_optional)],
) -> User:
if not user:
raise HTTPException(status.HTTP_401_UNAUTHORIZED, detail="未登录,请先飞书登录")
return user
def require_permission(permission: str):
def _dep(user: Annotated[User, Depends(get_current_user)]) -> User:
if not user_has_permission(user, permission):
raise HTTPException(status.HTTP_403_FORBIDDEN, detail=f"缺少权限: {permission}")
return user
return _dep
def require_any_permission(*permissions: str):
def _dep(user: Annotated[User, Depends(get_current_user)]) -> User:
if any(user_has_permission(user, p) for p in permissions):
return user
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="权限不足")
return _dep
def can_submit_action(user: User, action: str) -> bool:
if user_has_permission(user, "write:approval_submit"):
return True
if action == "register_batch" and user_has_permission(user, "write:approval_submit:register"):
return True
return False

View File

@@ -0,0 +1,142 @@
"""飞书 OAuth 登录。"""
from __future__ import annotations
import secrets
import urllib.parse
from datetime import datetime, timedelta, timezone
from typing import Any
import httpx
from jose import JWTError, jwt
from as_platform.config import FEISHU_APP_ID, FEISHU_APP_SECRET, FEISHU_REDIRECT_URI, JWT_SECRET
FEISHU_AUTHORIZE_URL = "https://passport.feishu.cn/suite/passport/oauth/authorize"
FEISHU_TOKEN_URL = "https://open.feishu.cn/open-apis/authen/v1/access_token"
FEISHU_USER_URL = "https://open.feishu.cn/open-apis/authen/v1/user_info"
FEISHU_TENANT_TOKEN_URL = "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal"
FEISHU_CONTACT_USER_URL = "https://open.feishu.cn/open-apis/contact/v3/users/{user_id}"
STATE_ALG = "HS256"
STATE_EXPIRE_MINUTES = 10
def is_feishu_configured() -> bool:
return bool(FEISHU_APP_ID and FEISHU_APP_SECRET)
def build_authorize_url() -> tuple[str, str]:
# 使用签名 state避免服务重启/多进程导致内存 state 丢失
now = datetime.now(timezone.utc)
state = jwt.encode(
{
"nonce": secrets.token_urlsafe(12),
"iat": int(now.timestamp()),
"exp": int((now + timedelta(minutes=STATE_EXPIRE_MINUTES)).timestamp()),
"typ": "feishu_oauth_state",
},
JWT_SECRET,
algorithm=STATE_ALG,
)
params = {
"client_id": FEISHU_APP_ID,
"redirect_uri": FEISHU_REDIRECT_URI,
"response_type": "code",
"state": state,
}
return f"{FEISHU_AUTHORIZE_URL}?{urllib.parse.urlencode(params)}", state
def verify_state(state: str) -> bool:
try:
payload = jwt.decode(state, JWT_SECRET, algorithms=[STATE_ALG])
return payload.get("typ") == "feishu_oauth_state"
except JWTError:
return False
def exchange_code(code: str) -> dict[str, Any]:
"""用授权码换 user_access_token 并拉取用户信息。"""
with httpx.Client(timeout=30.0) as client:
token_resp = client.post(
FEISHU_TOKEN_URL,
json={
"grant_type": "authorization_code",
"code": code,
"app_id": FEISHU_APP_ID,
"app_secret": FEISHU_APP_SECRET,
},
)
token_resp.raise_for_status()
token_data = token_resp.json()
if token_data.get("code") != 0:
raise RuntimeError(token_data.get("msg") or "飞书 token 交换失败")
access_token = token_data["data"]["access_token"]
user_resp = client.get(
FEISHU_USER_URL,
headers={"Authorization": f"Bearer {access_token}"},
)
user_resp.raise_for_status()
user_data = user_resp.json()
if user_data.get("code") != 0:
raise RuntimeError(user_data.get("msg") or "飞书用户信息获取失败")
info = user_data["data"]
department_ids: list[str] = []
user_id = info.get("user_id")
tenant_key = info.get("tenant_key")
open_id = info.get("open_id") or info.get("openId")
if open_id:
try:
tenant_access_token = _get_tenant_access_token(client)
contact_user = _get_contact_user_profile(client, tenant_access_token, open_id)
user_id = contact_user.get("user_id") or user_id
tenant_key = contact_user.get("tenant_key") or tenant_key
raw_department_ids = contact_user.get("department_ids")
if isinstance(raw_department_ids, list):
department_ids = [str(x) for x in raw_department_ids if x]
except Exception:
# 联系人接口失败时不阻断登录
department_ids = []
return {
"open_id": open_id,
"union_id": info.get("union_id") or info.get("unionId"),
"user_id": user_id,
"tenant_key": tenant_key,
"department_ids": department_ids,
"name": info.get("name") or info.get("en_name") or "飞书用户",
"email": info.get("email") or info.get("enterprise_email"),
"avatar_url": info.get("avatar_url") or info.get("avatar_big"),
}
def _get_tenant_access_token(client: httpx.Client) -> str:
resp = client.post(
FEISHU_TENANT_TOKEN_URL,
json={"app_id": FEISHU_APP_ID, "app_secret": FEISHU_APP_SECRET},
)
resp.raise_for_status()
data = resp.json()
if data.get("code") != 0:
raise RuntimeError(data.get("msg") or "飞书 tenant token 获取失败")
token = data.get("tenant_access_token")
if not token:
raise RuntimeError("飞书 tenant token 为空")
return token
def _get_contact_user_profile(
client: httpx.Client, tenant_access_token: str, open_id: str
) -> dict[str, Any]:
resp = client.get(
FEISHU_CONTACT_USER_URL.format(user_id=urllib.parse.quote(open_id, safe="")),
params={"user_id_type": "open_id", "department_id_type": "open_department_id"},
headers={"Authorization": f"Bearer {tenant_access_token}"},
)
resp.raise_for_status()
data = resp.json()
if data.get("code") != 0:
raise RuntimeError(data.get("msg") or "飞书联系人信息获取失败")
return data.get("data", {}).get("user", {})

View File

@@ -0,0 +1,29 @@
"""JWT 令牌。"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any
from jose import JWTError, jwt
from as_platform.config import JWT_EXPIRE_HOURS, JWT_SECRET
ALGORITHM = "HS256"
def create_access_token(user_id: int, extra: dict[str, Any] | None = None) -> str:
payload = {
"sub": str(user_id),
"exp": datetime.now(timezone.utc) + timedelta(hours=JWT_EXPIRE_HOURS),
"iat": datetime.now(timezone.utc),
}
if extra:
payload.update(extra)
return jwt.encode(payload, JWT_SECRET, algorithm=ALGORITHM)
def decode_access_token(token: str) -> dict[str, Any] | None:
try:
return jwt.decode(token, JWT_SECRET, algorithms=[ALGORITHM])
except JWTError:
return None

View File

@@ -0,0 +1,68 @@
"""用户服务。"""
from __future__ import annotations
import json
from typing import Any
from sqlalchemy.orm import Session, joinedload
from as_platform.db.init_db import assign_default_role
from as_platform.db.models import Role, User
def get_user_by_id(db: Session, user_id: int) -> User | None:
return (
db.query(User)
.options(joinedload(User.roles).joinedload(Role.permissions))
.filter(User.id == user_id)
.first()
)
def upsert_feishu_user(db: Session, info: dict[str, Any]) -> User:
open_id = info.get("open_id")
user = db.query(User).filter(User.feishu_open_id == open_id).first()
if not user:
user = User(feishu_open_id=open_id)
db.add(user)
user.feishu_union_id = info.get("union_id") or user.feishu_union_id
user.feishu_user_id = info.get("user_id") or user.feishu_user_id
user.feishu_tenant_key = info.get("tenant_key") or user.feishu_tenant_key
department_ids = info.get("department_ids")
if isinstance(department_ids, list):
user.feishu_department_ids_json = json.dumps(department_ids, ensure_ascii=False)
user.name = info.get("name") or user.name
user.email = info.get("email") or user.email
user.avatar_url = info.get("avatar_url") or user.avatar_url
user.is_active = True
db.flush()
assign_default_role(db, user)
db.refresh(user)
return get_user_by_id(db, user.id) # type: ignore
def get_or_create_dev_user(db: Session, name: str = "开发用户") -> User:
user = db.query(User).filter(User.name == name, User.feishu_open_id.is_(None)).first()
if not user:
user = User(name=name, email="dev@local")
db.add(user)
db.flush()
admin = db.query(Role).filter_by(code="admin").first()
if admin:
user.roles = [admin]
db.flush()
return get_user_by_id(db, user.id) # type: ignore
def list_users(db: Session) -> list[User]:
return db.query(User).options(joinedload(User.roles)).order_by(User.id).all()
def set_user_roles(db: Session, user_id: int, role_codes: list[str]) -> User | None:
user = get_user_by_id(db, user_id)
if not user:
return None
roles = db.query(Role).filter(Role.code.in_(role_codes)).all()
user.roles = roles
db.flush()
return user

View File

@@ -0,0 +1,71 @@
"""华胥卡车主动安全AEB平台根配置。"""
from __future__ import annotations
import os
from pathlib import Path
from urllib.parse import quote_plus
AS_ROOT = Path(__file__).resolve().parent.parent.parent
WORKSPACE = AS_ROOT
PLATFORM_DIR = AS_ROOT / "platform"
PLATFORM_WEB = PLATFORM_DIR / "web" / "dist"
ALGORITHMS_DIR = AS_ROOT / "algorithms"
DATASETS_DIR = AS_ROOT / "datasets"
ALGORITHMS_REGISTRY = ALGORITHMS_DIR / "registry.yaml"
MANIFESTS = AS_ROOT / "manifests"
JOB_LOG = MANIFESTS / "job_log.jsonl"
APPROVAL_QUEUE = MANIFESTS / "approval_queue.jsonl"
TRACE_LOG = MANIFESTS / "trace_log.jsonl"
GRAPH_STATE_DIR = MANIFESTS / "graph_state"
ENGINES_REGISTRY = ALGORITHMS_REGISTRY
SQLITE_LEGACY_PATH = MANIFESTS / "platform.db"
# ── 数据库(默认 PostgreSQL──
DB_HOST = os.environ.get("AS_DB_HOST", "127.0.0.1")
DB_PORT = os.environ.get("AS_DB_PORT", "5432")
DB_USER = os.environ.get("AS_DB_USER", "as_platform")
DB_PASSWORD = os.environ.get("AS_DB_PASSWORD", "as_platform")
DB_NAME = os.environ.get("AS_DB_NAME", "as_platform")
def build_database_url() -> str:
explicit = os.environ.get("AS_DATABASE_URL", "").strip()
if explicit:
return explicit
pwd = quote_plus(DB_PASSWORD)
return f"postgresql+psycopg2://{DB_USER}:{pwd}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
DATABASE_URL = build_database_url()
IS_POSTGRES = DATABASE_URL.startswith("postgresql")
IS_SQLITE = DATABASE_URL.startswith("sqlite")
# ── RedisJob 队列 / 事件docker compose 默认 redis:6379──
REDIS_URL = os.environ.get("AS_REDIS_URL", "redis://127.0.0.1:6379/0")
JOB_QUEUE_KEY = os.environ.get("AS_JOB_QUEUE_KEY", "as:job_queue")
# thread: API 进程内执行(本机 run_local.sh| worker: 推 Redis由 worker 容器/脚本消费
JOB_EXECUTOR = os.environ.get("AS_JOB_EXECUTOR", "thread")
# ── 认证 / 飞书 ──
JWT_SECRET = os.environ.get("AS_JWT_SECRET", "change-me-in-production")
JWT_EXPIRE_HOURS = int(os.environ.get("AS_JWT_EXPIRE_HOURS", "168"))
FEISHU_APP_ID = os.environ.get("FEISHU_APP_ID", "")
FEISHU_APP_SECRET = os.environ.get("FEISHU_APP_SECRET", "")
FEISHU_REDIRECT_URI = os.environ.get(
"FEISHU_REDIRECT_URI",
"http://127.0.0.1:8787/api/v1/auth/feishu/callback",
)
FRONTEND_URL = os.environ.get("AS_FRONTEND_URL", "http://127.0.0.1:8787")
DEV_AUTH_ENABLED = os.environ.get("AS_DEV_AUTH", "").lower() in ("1", "true", "yes")
FEISHU_ADMIN_OPEN_IDS = {
x.strip() for x in os.environ.get("FEISHU_ADMIN_OPEN_IDS", "").split(",") if x.strip()
}
FEISHU_ADMIN_DEPARTMENT_IDS = {
x.strip() for x in os.environ.get("FEISHU_ADMIN_DEPARTMENT_IDS", "").split(",") if x.strip()
}
# ── 功能开关 ──
# 车道线 catalog 质检统计mask 抽样,较慢);暂时默认关闭
LANE_DATA_VIZ_ENABLED = os.environ.get("AS_LANE_DATA_VIZ", "").lower() in ("1", "true", "yes")

View File

@@ -0,0 +1,37 @@
from as_platform.data.batch import META_FILENAME
from as_platform.data.core import (
get_catalog,
get_pending_report,
load_wf,
proj_root,
register_batch,
resolve_pack,
resolve_pack_dir,
)
from as_platform.data.ingest import inspect_uploaded_dataset
from as_platform.data.lake import (
analyze_uploaded_candidate,
create_uploaded_candidate,
get_candidate,
link_candidate_analysis_job,
list_candidates,
write_candidate_upload,
)
__all__ = [
"META_FILENAME",
"get_pending_report",
"get_catalog",
"register_batch",
"load_wf",
"proj_root",
"resolve_pack",
"resolve_pack_dir",
"inspect_uploaded_dataset",
"create_uploaded_candidate",
"write_candidate_upload",
"list_candidates",
"get_candidate",
"link_candidate_analysis_job",
"analyze_uploaded_candidate",
]

View File

@@ -0,0 +1,161 @@
"""批次元数据 batch.meta.yaml 与目录结构推断。"""
from __future__ import annotations
from pathlib import Path
from typing import Any
import yaml
META_FILENAME = "batch.meta.yaml"
SCHEMA = "huaxu-batch-v1"
IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".JPG", ".JPEG", ".PNG"}
def count_images(dir_path: Path) -> int:
if not dir_path.is_dir():
return 0
n = 0
for p in dir_path.rglob("*"):
if p.is_file() and p.suffix in IMG_EXTS:
n += 1
return n
def count_label_files(dir_path: Path) -> int:
if not dir_path.is_dir():
return 0
n = 0
for p in dir_path.rglob("*"):
if p.is_file() and p.suffix.lower() in (".txt", ".xml"):
n += 1
return n
def dms_has_images(batch_dir: Path) -> bool:
for sub in ("images", "images/train"):
if count_images(batch_dir / sub) > 0:
return True
return False
def dms_has_labels(batch_dir: Path) -> bool:
for sub in ("labels", "labels/train"):
d = batch_dir / sub
if d.is_dir() and count_label_files(d) > 0:
return True
return False
def infer_dms_stage(batch_dir: Path) -> str:
has_img = dms_has_images(batch_dir)
has_lab = dms_has_labels(batch_dir)
if has_img and has_lab:
return "returned"
if has_img:
return "raw_pool"
return "raw_pool"
def infer_lane_stage(path: Path) -> str:
if (path / "list" / "train_gt.txt").is_file():
return "ingested"
if (path / "train_val_gt.txt").is_file():
return "returned"
if any(path.glob("**/train_val_gt.txt")):
return "returned"
if count_images(path) > 0:
return "raw_pool"
return "raw_pool"
def read_meta(batch_dir: Path) -> dict[str, Any] | None:
p = batch_dir / META_FILENAME
if not p.is_file():
return None
try:
data = yaml.safe_load(p.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else None
except Exception:
return None
def write_meta(batch_dir: Path, data: dict[str, Any]) -> Path:
batch_dir.mkdir(parents=True, exist_ok=True)
data.setdefault("schema", SCHEMA)
p = batch_dir / META_FILENAME
p.write_text(
yaml.dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False),
encoding="utf-8",
)
return p
def enrich_batch(
batch_dir: Path,
*,
project: str,
task: str | None = None,
pack: str | None = None,
batch: str,
location: str,
) -> dict[str, Any]:
meta = read_meta(batch_dir) or {}
if project == "dms":
stage = meta.get("stage") or infer_dms_stage(batch_dir)
img_n = meta.get("counts", {}).get("images")
lab_n = meta.get("counts", {}).get("labels")
if img_n is None:
img_n = count_images(batch_dir / "images") + count_images(batch_dir / "images" / "train")
if lab_n is None:
lab_n = count_label_files(batch_dir / "labels") + count_label_files(batch_dir / "labels" / "train")
fmt = meta.get("format", "yolo")
else:
stage = meta.get("stage") or infer_lane_stage(batch_dir)
img_n = meta.get("counts", {}).get("images") or count_images(batch_dir)
lab_n = meta.get("counts", {}).get("labels")
fmt = meta.get("format", "ufld_archive")
if (batch_dir / "list" / "train_gt.txt").is_file():
try:
lab_n = sum(1 for _ in (batch_dir / "list" / "train_gt.txt").open(encoding="utf-8"))
except OSError:
lab_n = lab_n or 0
return {
"project": project,
"task": task or meta.get("task"),
"pack": pack or meta.get("pack"),
"batch": batch,
"stage": stage,
"location": location,
"path": str(batch_dir.resolve()),
"engineer": meta.get("engineer"),
"format": fmt,
"counts": {"images": img_n, "labels": lab_n},
"has_meta": bool(meta),
"next_cli": suggest_cli(project, task, pack, batch, stage, location),
}
def suggest_cli(
project: str,
task: str | None,
pack: str | None,
batch: str,
stage: str,
location: str,
) -> str:
if project == "dms":
p = pack or "dms_v2"
t = task or "<task>"
if location == "inbox" and stage == "returned":
return f"python as.py build dms {t} --pack {p} --batch {batch}"
if location == "sources" and stage == "returned":
return f"python as.py build dms {t} --pack {p} --all-sources"
if stage == "raw_pool":
return f"# 送标完成后放入 labels或: python as.py register-batch dms {t} --batch {batch} --stage returned"
return f"python as.py build dms {t} --pack {p}"
if stage == "returned":
return "python as.py add lane --src <archive> --engineer <name> --date YYYYMMDD"
if stage == "ingested":
return f"python as.py enable lane {pack or '<pack>'} && python as.py build lane"
return "python as.py add lane --src <path> --engineer <name> --date YYYYMMDD"

View File

@@ -0,0 +1,246 @@
"""Catalog cache: memory/disk + cheap directory-change invalidation."""
from __future__ import annotations
import csv
import json
import os
import time
from pathlib import Path
from typing import Any
from as_platform.config import WORKSPACE
CATALOG_CACHE_FILE = WORKSPACE / "manifests" / "catalog_cache.json"
CATALOG_CACHE_TTL_SEC = int(os.environ.get("AS_CATALOG_CACHE_TTL_SEC", "300"))
CATALOG_USE_REPORTS = os.environ.get("AS_CATALOG_USE_REPORTS", "1").lower() in ("1", "true", "yes")
CATALOG_CACHE_VERSION = 3
REPORTS_DIR = WORKSPACE / "reports"
DMS_SUMMARY_CSV = REPORTS_DIR / "dms_task_image_summary.csv"
DMS_CLASS_CSV = REPORTS_DIR / "dms_task_class_image_counts.csv"
_CATALOG_MEM_CACHE: dict[str, Any] | None = None
def invalidate_catalog_cache() -> None:
global _CATALOG_MEM_CACHE
_CATALOG_MEM_CACHE = None
if CATALOG_CACHE_FILE.is_file():
try:
CATALOG_CACHE_FILE.unlink()
except OSError:
pass
def _dir_fingerprint(path: Path, *, scan_children: bool = True) -> dict[str, Any]:
if not path.exists():
return {"path": str(path), "missing": True}
try:
st = path.stat()
fp: dict[str, Any] = {
"path": str(path),
"mtime_ns": st.st_mtime_ns,
"size": st.st_size,
}
if scan_children and path.is_dir():
children: list[dict[str, Any]] = []
try:
with os.scandir(path) as it:
for entry in it:
if entry.name.startswith("."):
continue
try:
est = entry.stat(follow_symlinks=False)
children.append(
{
"name": entry.name,
"mtime_ns": est.st_mtime_ns,
"is_dir": entry.is_dir(follow_symlinks=False),
}
)
except OSError:
children.append({"name": entry.name, "error": True})
except OSError:
fp["scan_error"] = True
children.sort(key=lambda x: x.get("name", ""))
fp["children"] = children
return fp
except OSError:
return {"path": str(path), "error": True}
def build_catalog_signature(wf: dict, proj_root_fn) -> dict[str, Any]:
"""Cheap signature: config files + inbox/pack directory mtimes (auto-invalidate on drop)."""
from as_platform.data.core import _pack_registry_path, load_pack_registry
files: list[dict[str, Any]] = []
for rel in ("workflow.registry.yaml",):
p = WORKSPACE / rel
try:
st = p.stat()
files.append({"path": str(p), "mtime_ns": st.st_mtime_ns, "size": st.st_size})
except FileNotFoundError:
files.append({"path": str(p), "missing": True})
dirs: list[dict[str, Any]] = []
for pname in ("dms", "lane"):
root = proj_root_fn(wf, pname)
dirs.append(_dir_fingerprint(root, scan_children=False))
dirs.append(_dir_fingerprint(root / "inbox"))
try:
packs_reg = load_pack_registry(pname, root, wf)
except (FileNotFoundError, json.JSONDecodeError, OSError):
packs_reg = {"packs": []}
dirs.append(_dir_fingerprint(root / "packs", scan_children=False))
for p in packs_reg.get("packs", []):
pack_path = root / p.get("path", p.get("name", ""))
dirs.append(_dir_fingerprint(pack_path, scan_children=False))
if pname == "dms":
for cfg_path in (
root / wf["projects"]["dms"]["registry"],
root / "manifests" / "dataset_class_summary.txt",
_pack_registry_path("dms", root, wf),
):
try:
st = cfg_path.stat()
files.append({"path": str(cfg_path), "mtime_ns": st.st_mtime_ns, "size": st.st_size})
except FileNotFoundError:
files.append({"path": str(cfg_path), "missing": True})
reg_path = root / wf["projects"]["dms"]["registry"]
if reg_path.is_file():
import yaml
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
for task in (reg.get("tasks") or {}).keys():
dirs.append(_dir_fingerprint(root / "inbox" / task))
if pname == "lane":
dirs.append(_dir_fingerprint(_pack_registry_path("lane", root, wf), scan_children=False))
for csv_path in (DMS_SUMMARY_CSV, DMS_CLASS_CSV):
try:
st = csv_path.stat()
files.append({"path": str(csv_path), "mtime_ns": st.st_mtime_ns, "size": st.st_size})
except FileNotFoundError:
files.append({"path": str(csv_path), "missing": True})
return {"files": files, "dirs": dirs}
def load_disk_cache() -> dict[str, Any] | None:
if not CATALOG_CACHE_FILE.is_file():
return None
try:
return json.loads(CATALOG_CACHE_FILE.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return None
def save_disk_cache(payload: dict[str, Any]) -> None:
CATALOG_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
CATALOG_CACHE_FILE.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
def _catalog_has_empty_bbox(catalog: dict[str, Any]) -> bool:
for task in (catalog.get("dms") or {}).values():
for pack in task.get("packs") or []:
boxes = int(pack.get("total_boxes") or 0)
pts = pack.get("bbox_points") or []
if boxes > 0 and not pts:
return True
return False
def get_cached_catalog(signature: dict[str, Any], *, refresh: bool = False) -> tuple[dict[str, Any] | None, dict[str, Any]]:
"""Return (catalog_data, meta). meta describes cache hit/miss."""
global _CATALOG_MEM_CACHE
now = time.time()
meta: dict[str, Any] = {"cached": False, "source": "scan"}
if refresh:
_CATALOG_MEM_CACHE = None
return None, meta
for source, cache in (("memory", _CATALOG_MEM_CACHE), ("disk", load_disk_cache() if not _CATALOG_MEM_CACHE else None)):
if not cache:
continue
age = now - float(cache.get("generated_at_ts", 0.0))
if cache.get("signature") != signature:
continue
if cache.get("version") != CATALOG_CACHE_VERSION:
continue
data = cache.get("data") or {}
if _catalog_has_empty_bbox(data):
continue
if age > CATALOG_CACHE_TTL_SEC:
continue
if source == "disk":
_CATALOG_MEM_CACHE = cache
meta = {
"cached": True,
"cache_source": source,
"cache_age_sec": round(age, 1),
"generated_at_ts": cache.get("generated_at_ts"),
"build_source": cache.get("build_source", "scan"),
}
return cache.get("data", {}), meta
return None, meta
def store_catalog_cache(signature: dict[str, Any], data: dict[str, Any], *, build_source: str = "scan") -> dict[str, Any]:
global _CATALOG_MEM_CACHE
now = time.time()
payload = {
"version": CATALOG_CACHE_VERSION,
"generated_at_ts": now,
"signature": signature,
"build_source": build_source,
"data": data,
}
_CATALOG_MEM_CACHE = payload
save_disk_cache(payload)
return payload
def load_dms_reports() -> tuple[dict[tuple[str, str], dict[str, int]], dict[str, dict[str, int]]] | None:
"""Parse precomputed CSV reports: (task, pack) -> splits, task -> class_counts."""
if not CATALOG_USE_REPORTS or not DMS_SUMMARY_CSV.is_file():
return None
splits: dict[tuple[str, str], dict[str, int]] = {}
try:
with DMS_SUMMARY_CSV.open(encoding="utf-8") as f:
for row in csv.DictReader(f):
task = row.get("任务", "").strip()
pack = row.get("数据包", "").strip() or "default"
if not task:
continue
splits[(task, pack)] = {
"train": int(row.get("训练集图片") or 0),
"val": int(row.get("验证集图片") or 0),
"test": int(row.get("测试集图片") or 0),
"total": int(row.get("图片总数") or 0),
}
except (OSError, ValueError, csv.Error):
return None
class_by_task: dict[str, dict[str, int]] = {}
if DMS_CLASS_CSV.is_file():
try:
with DMS_CLASS_CSV.open(encoding="utf-8") as f:
for row in csv.DictReader(f):
task = row.get("任务", "").strip()
cls_name = row.get("类别名", "").strip()
if not task or not cls_name:
continue
try:
cnt = int(row.get("含该类别图片数") or 0)
except ValueError:
continue
class_by_task.setdefault(task, {})[cls_name] = cnt
except (OSError, ValueError, csv.Error):
pass
if not splits:
return None
return splits, class_by_task

View File

@@ -0,0 +1,826 @@
"""平台共享逻辑pending、catalog、register-batch。"""
from __future__ import annotations
import json
import math
import os
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import yaml
from as_platform.config import WORKSPACE, LANE_DATA_VIZ_ENABLED
from as_platform.data.batch import META_FILENAME, enrich_batch, write_meta
from as_platform.data.catalog_cache import (
build_catalog_signature,
get_cached_catalog,
invalidate_catalog_cache,
load_dms_reports,
store_catalog_cache,
)
MAX_LABEL_FILES_PER_PACK = 2000
MAX_BBOX_POINTS_PER_PACK = 1500
MAX_LANE_MASK_SAMPLES_PER_PACK = 500
LANE_Y_BINS = 12
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
def load_wf() -> dict:
return yaml.safe_load((WORKSPACE / "workflow.registry.yaml").read_text(encoding="utf-8"))
def proj_root(wf: dict, name: str) -> Path:
return (WORKSPACE / wf["projects"][name]["root"]).resolve()
def load_pack_registry(project: str, root: Path, wf: dict) -> dict:
pcfg = wf["projects"][project]
reg_file = root / pcfg.get("packs_registry", "datasets_registry.json")
if reg_file.suffix in (".yaml", ".yml"):
return yaml.safe_load(reg_file.read_text(encoding="utf-8"))
return json.loads(reg_file.read_text(encoding="utf-8"))
def _pack_registry_path(project: str, root: Path, wf: dict) -> Path:
pcfg = wf["projects"][project]
return root / pcfg.get("packs_registry", "datasets_registry.json")
def resolve_pack(project: str, root: Path, wf: dict, name: str) -> str:
reg = load_pack_registry(project, root, wf)
name = reg.get("aliases", {}).get(name, name)
for p in reg.get("packs", []):
if p.get("name") == name:
return p.get("path", name)
if (root / name).is_dir():
return name
known = [p.get("name") for p in reg.get("packs", [])]
raise ValueError(f"[{project}] 未知包: {name},已登记: {known}")
def resolve_pack_dir(project: str, root: Path, wf: dict, name: str) -> Path:
return (root / resolve_pack(project, root, wf, name)).resolve()
def _read_jsonl_tail(path: Path, n: int = 10) -> list[dict]:
if not path.is_file():
return []
lines = path.read_text(encoding="utf-8").strip().splitlines()
out = []
for line in lines[-n:]:
try:
out.append(json.loads(line))
except json.JSONDecodeError:
pass
return out
def get_pending_report(wf: dict | None = None) -> dict[str, Any]:
wf = wf or load_wf()
report: dict[str, Any] = {
"workspace": str(WORKSPACE),
"updated_at": datetime.now(timezone.utc).isoformat(),
"projects": {},
"batches": [],
}
for pname, pcfg in wf["projects"].items():
root = proj_root(wf, pname)
active = set(pcfg.get("active_packs", []))
try:
reg_all = load_pack_registry(pname, root, wf)
except (FileNotFoundError, json.JSONDecodeError):
reg_all = {"packs": []}
all_names = {p["name"] for p in reg_all.get("packs", [])}
not_active = sorted(all_names - active)
proj: dict[str, Any] = {
"root": str(root),
"active_packs": list(active),
"not_enabled": not_active,
"tasks": {},
"task_defs": {},
}
if pname == "dms":
reg_path = root / pcfg["registry"]
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
src_sub = (reg.get("ingest") or {}).get("sources_subdir", "sources")
ingest_log = root / "manifests" / "ingest_log.jsonl"
proj["recent_ingest"] = _read_jsonl_tail(ingest_log, 10)
for task, tcfg in reg.get("tasks", {}).items():
proj["task_defs"][task] = {
"type": tcfg.get("type"),
"nc": tcfg.get("nc"),
"names": tcfg.get("names"),
"task_dir": tcfg.get("task_dir", task),
}
inbox_batches: list[str] = []
ib = root / "inbox" / task
if ib.is_dir():
inbox_batches = [
d.name for d in ib.iterdir()
if d.is_dir() and not d.name.startswith(".")
]
sources_pending: dict[str, list[str]] = {}
for pack_name in all_names:
try:
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
except ValueError:
continue
src_root = pack_dir / tcfg["task_dir"] / src_sub
if src_root.is_dir():
batches = [
d.name for d in src_root.iterdir()
if d.is_dir()
and d.name not in ("_ingested", "_merged")
and not d.name.startswith(".")
]
if batches:
sources_pending[pack_name] = batches
proj["tasks"][task] = {
"inbox": inbox_batches,
"sources": sources_pending,
}
for batch_name in inbox_batches:
batch_dir = ib / batch_name
report["batches"].append(
enrich_batch(
batch_dir,
project="dms",
task=task,
pack=None,
batch=batch_name,
location="inbox",
)
)
for pack_name, batch_list in sources_pending.items():
try:
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
except ValueError:
continue
src_root = pack_dir / tcfg["task_dir"] / src_sub
for batch_name in batch_list:
batch_dir = src_root / batch_name
report["batches"].append(
enrich_batch(
batch_dir,
project="dms",
task=task,
pack=pack_name,
batch=batch_name,
location="sources",
)
)
if pname == "lane":
proj["packs"] = {}
for pack_name in all_names:
try:
path = resolve_pack("lane", root, wf, pack_name)
except ValueError:
continue
pack_path = root / path
train_lines = 0
tg = pack_path / "list" / "train_gt.txt"
if tg.is_file():
train_lines = sum(1 for _ in tg.open(encoding="utf-8"))
proj["packs"][pack_name] = {
"path": path,
"train_lines": train_lines,
"enabled": pack_name in active,
}
if pack_name in not_active and pack_path.is_dir():
report["batches"].append(
enrich_batch(
pack_path,
project="lane",
task=None,
pack=pack_name,
batch=path,
location="pack",
)
)
for child in sorted(root.iterdir()) if root.is_dir() else []:
if not child.is_dir() or child.name.startswith("."):
continue
if child.name in ("lists_merged", "scripts", "inbox"):
continue
if not child.name.startswith("DATASET-AddBy-"):
continue
if any(
p.get("path") == child.name or p.get("name") == child.name
for p in reg_all.get("packs", [])
):
continue
report["batches"].append(
enrich_batch(
child,
project="lane",
task=None,
pack=None,
batch=child.name,
location="unregistered",
)
)
inbox_lane = root / "inbox"
if inbox_lane.is_dir():
for batch_dir in sorted(inbox_lane.iterdir()):
if batch_dir.is_dir() and not batch_dir.name.startswith("."):
report["batches"].append(
enrich_batch(
batch_dir,
project="lane",
task=None,
pack=None,
batch=batch_dir.name,
location="inbox",
)
)
report["projects"][pname] = proj
return report
def _parse_class_summary(text: str) -> dict[str, dict[str, int]]:
"""解析 dataset_class_summary.txt 按 task 的类统计。"""
by_task: dict[str, dict[str, int]] = {}
current_task: str | None = None
for line in text.splitlines():
line = line.strip()
if not line:
continue
if line.endswith(":") and " " not in line.rstrip(":"):
current_task = line.rstrip(":")
by_task.setdefault(current_task, {})
continue
if current_task and ":" in line:
parts = line.split(":", 1)
cls_name = parts[0].strip()
try:
count = int(re.search(r"\d+", parts[1]).group()) # type: ignore
by_task[current_task][cls_name] = count
except (AttributeError, ValueError):
pass
return by_task
def _class_name_map(tcfg: dict[str, Any]) -> dict[int, str]:
names = tcfg.get("names")
if isinstance(names, list):
return {idx: str(name) for idx, name in enumerate(names)}
if isinstance(names, dict):
out: dict[int, str] = {}
for k, v in names.items():
try:
out[int(k)] = str(v)
except (TypeError, ValueError):
continue
return out
return {}
def _count_images_in_dir(img_dir: Path) -> int:
if not img_dir.is_dir():
return 0
total = 0
try:
with os.scandir(img_dir) as it:
for entry in it:
if not entry.is_file(follow_symlinks=False):
continue
if Path(entry.name).suffix.lower() in IMAGE_EXTS:
total += 1
except OSError:
return 0
return total
def _count_split_images(task_data: Path) -> dict[str, int]:
counts = {
"train": _count_images_in_dir(task_data / "images" / "train"),
"val": _count_images_in_dir(task_data / "images" / "val"),
"test": _count_images_in_dir(task_data / "images" / "test"),
}
if sum(counts.values()) == 0:
flat = _count_images_in_dir(task_data / "images")
if flat:
counts["train"] = flat
return counts
def _iter_label_files(label_dirs: list[Path]):
for label_dir in label_dirs:
if not label_dir.is_dir():
continue
try:
with os.scandir(label_dir) as it:
stack = [entry.path for entry in it if entry.is_dir(follow_symlinks=False)]
files = [entry.path for entry in it if entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt")]
except OSError:
continue
for fp in files:
yield Path(fp)
while stack:
current = stack.pop()
try:
with os.scandir(current) as it:
for entry in it:
if entry.is_dir(follow_symlinks=False):
stack.append(entry.path)
elif entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt"):
yield Path(entry.path)
except OSError:
continue
def _label_dirs_for_task(task_data: Path) -> list[Path]:
return [task_data / "labels" / "train", task_data / "labels" / "val", task_data / "labels"]
def _parse_bbox_wh(parts: list[str]) -> list[float] | None:
if len(parts) < 5:
return None
try:
w = float(parts[3])
h = float(parts[4])
if 0.0 < w <= 1.0 and 0.0 < h <= 1.0:
return [round(w, 6), round(h, 6)]
except ValueError:
return None
return None
def _collect_bbox_points_sample(task_data: Path, *, max_points: int = MAX_BBOX_POINTS_PER_PACK) -> list[list[float]]:
"""Lightweight sample for scatter plot; does not scan images."""
bbox_points: list[list[float]] = []
remaining_files = MAX_LABEL_FILES_PER_PACK
for txt in _iter_label_files(_label_dirs_for_task(task_data)):
if len(bbox_points) >= max_points or remaining_files <= 0:
break
remaining_files -= 1
try:
for line in txt.read_text(encoding="utf-8", errors="ignore").splitlines():
if len(bbox_points) >= max_points:
break
line = line.strip()
if not line:
continue
wh = _parse_bbox_wh(line.split())
if wh:
bbox_points.append(wh)
except OSError:
continue
return bbox_points
def _collect_pack_label_distribution(task_data: Path, tcfg: dict[str, Any]) -> dict[str, Any]:
label_dirs = _label_dirs_for_task(task_data)
class_counts: dict[int, int] = {}
bbox_points: list[list[float]] = []
parsed_files = 0
sampled = False
remaining = MAX_LABEL_FILES_PER_PACK
for txt in _iter_label_files(label_dirs):
if remaining <= 0:
sampled = True
break
remaining -= 1
parsed_files += 1
try:
for line in txt.read_text(encoding="utf-8", errors="ignore").splitlines():
line = line.strip()
if not line:
continue
cls_token = line.split(maxsplit=1)[0]
cls_id = int(float(cls_token))
class_counts[cls_id] = class_counts.get(cls_id, 0) + 1
parts = line.split()
if len(bbox_points) < MAX_BBOX_POINTS_PER_PACK:
wh = _parse_bbox_wh(parts)
if wh:
bbox_points.append(wh)
except OSError:
continue
name_map = _class_name_map(tcfg)
by_name: dict[str, int] = {}
for cls_id, cnt in sorted(class_counts.items(), key=lambda x: x[1], reverse=True):
key = name_map.get(cls_id, f"class_{cls_id}")
by_name[key] = cnt
return {
"class_counts": by_name,
"label_files": parsed_files,
"sampled": sampled,
"total_boxes": sum(class_counts.values()),
"bbox_points": bbox_points,
}
def _histogram(values: list[float], bins: list[float]) -> list[dict[str, float]]:
if len(bins) < 2:
return []
counts = [0] * (len(bins) - 1)
for v in values:
for i in range(len(bins) - 1):
lo, hi = bins[i], bins[i + 1]
if (v >= lo and v < hi) or (i == len(bins) - 2 and v >= lo and v <= hi):
counts[i] += 1
break
return [
{"left": bins[i], "right": bins[i + 1], "count": counts[i]}
for i in range(len(counts))
]
def _extract_lane_mask_stats(mask_path: Path) -> dict[str, Any] | None:
try:
from PIL import Image # type: ignore
except ImportError:
return None
try:
img = Image.open(mask_path).convert("L")
except OSError:
return None
w, h = img.size
pix = img.load()
if pix is None:
return None
lane_bins: dict[int, list[dict[str, float]]] = {}
present_ids: set[int] = set()
for y in range(h):
by_id: dict[int, tuple[int, int]] = {}
for x in range(w):
lane_id = int(pix[x, y])
if lane_id <= 0:
continue
present_ids.add(lane_id)
if lane_id not in by_id:
by_id[lane_id] = (x, x)
else:
mn, mx = by_id[lane_id]
if x < mn:
mn = x
if x > mx:
mx = x
by_id[lane_id] = (mn, mx)
if not by_id:
continue
y_bin = min(LANE_Y_BINS - 1, int((y / max(1, h - 1)) * LANE_Y_BINS))
for lane_id, (mn, mx) in by_id.items():
bucket = lane_bins.setdefault(lane_id, [dict(min_x=1e9, max_x=-1e9, count=0) for _ in range(LANE_Y_BINS)])
cur = bucket[y_bin]
cur["min_x"] = min(cur["min_x"], float(mn))
cur["max_x"] = max(cur["max_x"], float(mx))
cur["count"] += 1
lengths: list[float] = []
curvatures: list[float] = []
for lane_id in sorted(present_ids):
bins = lane_bins.get(lane_id, [])
centers: list[tuple[float, float]] = []
for i, b in enumerate(bins):
if b["count"] <= 0:
continue
center_x = (b["min_x"] + b["max_x"]) / 2.0
center_y = ((i + 0.5) / LANE_Y_BINS) * h
centers.append((center_x, center_y))
if len(centers) < 2:
continue
length = 0.0
for i in range(1, len(centers)):
dx = centers[i][0] - centers[i - 1][0]
dy = centers[i][1] - centers[i - 1][1]
length += math.sqrt(dx * dx + dy * dy)
lengths.append(length)
if len(centers) >= 3:
second_diffs = []
xs = [c[0] for c in centers]
for i in range(1, len(xs) - 1):
second_diffs.append(abs(xs[i + 1] - 2 * xs[i] + xs[i - 1]))
if second_diffs:
curvatures.append(sum(second_diffs) / len(second_diffs))
return {
"lane_count": len(present_ids),
"lengths": lengths,
"curvatures": curvatures,
}
def _collect_lane_quality(pack_path: Path) -> dict[str, Any]:
list_files = [pack_path / "list" / "train_gt.txt", pack_path / "list" / "val_gt.txt"]
entries: list[Path] = []
for lf in list_files:
if not lf.is_file():
continue
try:
for line in lf.read_text(encoding="utf-8", errors="ignore").splitlines():
line = line.strip()
if not line:
continue
parts = line.split()
if len(parts) < 2:
continue
entries.append(pack_path / parts[1])
if len(entries) >= MAX_LANE_MASK_SAMPLES_PER_PACK:
break
except OSError:
continue
if len(entries) >= MAX_LANE_MASK_SAMPLES_PER_PACK:
break
lane_counts: list[float] = []
lane_lengths: list[float] = []
lane_curvatures: list[float] = []
processed = 0
for ann in entries:
s = _extract_lane_mask_stats(ann)
if not s:
continue
processed += 1
lane_counts.append(float(s["lane_count"]))
lane_lengths.extend(float(x) for x in s["lengths"])
lane_curvatures.extend(float(x) for x in s["curvatures"])
lane_count_hist: dict[str, int] = {}
for c in lane_counts:
key = str(int(c)) if c < 8 else "8+"
lane_count_hist[key] = lane_count_hist.get(key, 0) + 1
return {
"analyzed_frames": processed,
"lane_count_hist": lane_count_hist,
"length_hist": _histogram(lane_lengths, [0, 60, 120, 180, 240, 320, 420, 560, 760, 1024]),
"curvature_hist": _histogram(lane_curvatures, [0, 1, 2, 4, 6, 8, 12, 16, 24, 40]),
}
def _catalog_signature(wf: dict) -> dict[str, Any]:
return build_catalog_signature(wf, proj_root)
def _build_catalog(wf: dict, *, prefer_reports: bool = True) -> tuple[dict[str, Any], str]:
out: dict[str, Any] = {"workspace": str(WORKSPACE), "dms": {}, "lane": {}}
build_source = "scan"
reports = load_dms_reports() if prefer_reports else None
report_splits: dict[tuple[str, str], dict[str, int]] = {}
report_classes: dict[str, dict[str, int]] = {}
if reports:
report_splits, report_classes = reports
build_source = "reports"
root = proj_root(wf, "dms")
reg_path = root / wf["projects"]["dms"]["registry"]
if not reg_path.is_file():
out["dms_error"] = f"registry not found: {reg_path}"
if report_splits:
for (task, pack_name), rep in report_splits.items():
entry = out["dms"].setdefault(task, {
"type": "unknown",
"class_counts": report_classes.get(task, {}),
"packs": [],
})
entry["packs"].append({
"name": pack_name,
"enabled": False,
"train_images": rep.get("train", 0),
"val_images": rep.get("val", 0),
"test_images": rep.get("test", 0),
"class_counts": report_classes.get(task, {}),
"label_files": 0,
"total_boxes": sum(report_classes.get(task, {}).values()) if task in report_classes else 0,
"sampled": True,
"bbox_points": [],
})
reg = {"tasks": {}}
else:
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
try:
packs_reg = load_pack_registry("dms", root, wf)
except (FileNotFoundError, json.JSONDecodeError, OSError):
packs_reg = {"packs": []}
summary_path = root / "manifests" / "dataset_class_summary.txt"
class_by_task = {}
if summary_path.is_file():
class_by_task = _parse_class_summary(summary_path.read_text(encoding="utf-8"))
for task, tcfg in reg.get("tasks", {}).items():
entry: dict[str, Any] = {
"type": tcfg.get("type"),
"nc": tcfg.get("nc"),
"names": tcfg.get("names"),
"class_counts": class_by_task.get(task, {}),
"packs": [],
"drop_paths": {
"inbox": str((root / "inbox" / task).resolve()),
"sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "sources" / "<batch>").resolve()),
},
}
for p in packs_reg.get("packs", []):
pack_name = p["name"]
try:
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
except ValueError:
continue
task_data = pack_dir / tcfg.get("task_dir", task)
rep = report_splits.get((task, pack_name))
if rep:
split_counts = {"train": rep["train"], "val": rep["val"], "test": rep["test"]}
class_counts = report_classes.get(task, class_by_task.get(task, {}))
bbox_points = _collect_bbox_points_sample(task_data)
label_distribution = {
"class_counts": class_counts,
"label_files": 0,
"sampled": True,
"total_boxes": sum(class_counts.values()) if class_counts else 0,
"bbox_points": bbox_points,
}
else:
split_counts = _count_split_images(task_data)
label_distribution = _collect_pack_label_distribution(task_data, tcfg)
if not label_distribution["class_counts"] and task in report_classes:
label_distribution["class_counts"] = report_classes[task]
entry["packs"].append({
"name": pack_name,
"path": p.get("path"),
"role": p.get("role"),
"frozen": p.get("frozen", False),
"enabled": pack_name in wf["projects"]["dms"].get("active_packs", []),
"train_images": split_counts.get("train", 0),
"val_images": split_counts.get("val", 0),
"test_images": split_counts.get("test", 0),
"class_counts": label_distribution["class_counts"],
"label_files": label_distribution["label_files"],
"total_boxes": label_distribution["total_boxes"],
"sampled": label_distribution["sampled"],
"bbox_points": label_distribution["bbox_points"],
})
if not entry["class_counts"] and task in report_classes:
entry["class_counts"] = report_classes[task]
out["dms"][task] = entry
root = proj_root(wf, "lane")
try:
reg = load_pack_registry("lane", root, wf)
except (FileNotFoundError, json.JSONDecodeError, OSError):
reg = {"packs": []}
for p in reg.get("packs", []):
pack_name = p["name"]
path = p.get("path", pack_name)
pack_path = root / path
tg = pack_path / "list" / "train_gt.txt"
vg = pack_path / "list" / "val_gt.txt"
sg = pack_path / "list" / "test_gt.txt"
lane_quality = _collect_lane_quality(pack_path) if LANE_DATA_VIZ_ENABLED else {}
out["lane"][pack_name] = {
"path": path,
"role": p.get("role"),
"frozen": p.get("frozen", False),
"enabled": pack_name in wf["projects"]["lane"].get("active_packs", []),
"train_lines": sum(1 for _ in tg.open(encoding="utf-8")) if tg.is_file() else 0,
"val_lines": sum(1 for _ in vg.open(encoding="utf-8")) if vg.is_file() else 0,
"test_lines": sum(1 for _ in sg.open(encoding="utf-8")) if sg.is_file() else 0,
"drop_path": str(pack_path.resolve()),
"add_template": "python as.py add lane --src <archive> --engineer <name> --date YYYYMMDD",
"quality": lane_quality,
}
return out, build_source
def get_catalog(
wf: dict | None = None,
project: str | None = None,
task_or_pack: str | None = None,
*,
refresh: bool = False,
) -> dict:
wf = wf or load_wf()
sig = _catalog_signature(wf)
full_catalog, cache_meta = get_cached_catalog(sig, refresh=refresh)
if not full_catalog:
full_catalog, build_source = _build_catalog(wf, prefer_reports=not refresh)
store_catalog_cache(sig, full_catalog, build_source=build_source)
cache_meta = {"cached": False, "build_source": build_source}
result: dict[str, Any]
if project == "dms" and task_or_pack:
result = {"task": task_or_pack, **(full_catalog.get("dms", {}).get(task_or_pack, {}))}
elif project == "lane" and task_or_pack:
result = {"pack": task_or_pack, **(full_catalog.get("lane", {}).get(task_or_pack, {}))}
elif project in ("dms", "lane"):
result = {"workspace": full_catalog.get("workspace", str(WORKSPACE)), project: full_catalog.get(project, {})}
else:
result = dict(full_catalog)
if not task_or_pack and project is None:
result["_cache"] = cache_meta
return result
def warmup_catalog_cache() -> None:
"""Background warmup for faster first page load."""
try:
invalidate_catalog_cache()
get_catalog(refresh=True)
except Exception:
pass
def register_batch(
wf: dict | None,
project: str,
task: str | None,
batch: str,
*,
pack: str | None = None,
stage: str = "returned",
engineer: str | None = None,
location: str = "inbox",
) -> dict[str, Any]:
wf = wf or load_wf()
root = proj_root(wf, project)
if project == "dms":
if not task:
raise ValueError("dms register-batch 需要 task")
reg = yaml.safe_load((root / wf["projects"]["dms"]["registry"]).read_text(encoding="utf-8"))
if task not in reg.get("tasks", {}):
raise ValueError(f"未知 task: {task}")
tcfg = reg["tasks"][task]
if location == "sources":
if not pack:
raise ValueError("sources 位置需要 --pack")
pack_dir = resolve_pack_dir("dms", root, wf, pack)
src_sub = (reg.get("ingest") or {}).get("sources_subdir", "sources")
batch_dir = pack_dir / tcfg["task_dir"] / src_sub / batch
else:
batch_dir = root / "inbox" / task / batch
else:
if location == "pack" and pack:
try:
path = resolve_pack("lane", root, wf, pack)
batch_dir = root / path
except ValueError:
batch_dir = root / pack
else:
batch_dir = root / "inbox" / batch
if not batch_dir.is_dir():
raise FileNotFoundError(f"批次目录不存在: {batch_dir}")
data = {
"schema": "huaxu-batch-v1",
"project": project,
"task": task,
"pack": pack,
"batch": batch,
"stage": stage,
"engineer": engineer,
"registered_at": datetime.now(timezone.utc).isoformat(),
}
if project == "dms":
from as_platform.data.batch import count_images, count_label_files, dms_has_images
data["format"] = "yolo"
data["counts"] = {
"images": count_images(batch_dir / "images") + count_images(batch_dir / "images" / "train"),
"labels": count_label_files(batch_dir / "labels") + count_label_files(batch_dir / "labels" / "train"),
}
if not data["counts"]["images"] and dms_has_images(batch_dir):
data["counts"]["images"] = 1
else:
data["format"] = "ufld_archive"
tg = batch_dir / "list" / "train_gt.txt"
data["counts"] = {"images": 0, "labels": sum(1 for _ in tg.open()) if tg.is_file() else 0}
meta_path = write_meta(batch_dir, data)
invalidate_catalog_cache()
return {
"ok": True,
"meta_path": str(meta_path),
"batch": enrich_batch(
batch_dir,
project=project,
task=task,
pack=pack,
batch=batch,
location=location,
),
}

View File

@@ -0,0 +1,17 @@
from as_platform.data.ingest.base import IngestContext, IngestAdapter, NormalizedDataset
from as_platform.data.ingest.registry import (
UnknownFormatError,
available_formats,
detect_adapter,
inspect_uploaded_dataset,
)
__all__ = [
"IngestContext",
"IngestAdapter",
"NormalizedDataset",
"UnknownFormatError",
"available_formats",
"detect_adapter",
"inspect_uploaded_dataset",
]

View File

@@ -0,0 +1,46 @@
"""Data ingest adapter base abstractions."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any
@dataclass
class IngestContext:
project: str
task: str | None
source_path: Path
@dataclass
class NormalizedDataset:
format_id: str
project: str
task: str | None
source_path: str
split_counts: dict[str, int] = field(default_factory=dict)
sample_count: int = 0
annotation_count: int = 0
artifacts: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
extra: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return asdict(self)
class IngestAdapter(ABC):
"""Adapter interface for task-specific upload formats."""
format_id: str = "unknown"
projects: tuple[str, ...] = ()
@abstractmethod
def can_handle(self, ctx: IngestContext) -> bool:
raise NotImplementedError
@abstractmethod
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
raise NotImplementedError

View File

@@ -0,0 +1,88 @@
"""DMS COCO-format adapter."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
COCO_NAMES = ("instances_train.json", "instances_val.json", "instances_test.json", "annotations.json")
def _read_json(path: Path) -> dict[str, Any] | None:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return None
class DmsCocoAdapter(IngestAdapter):
format_id = "dms_coco"
projects = ("dms",)
def _find_coco_files(self, root: Path) -> list[Path]:
files: list[Path] = []
for name in COCO_NAMES:
p = root / "annotations" / name
if p.is_file():
files.append(p)
for name in COCO_NAMES:
p = root / name
if p.is_file():
files.append(p)
return files
def can_handle(self, ctx: IngestContext) -> bool:
root = ctx.source_path
return len(self._find_coco_files(root)) > 0
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
root = ctx.source_path
files = self._find_coco_files(root)
split_counts = {"train": 0, "val": 0, "test": 0}
ann_count = 0
categories: set[str] = set()
warnings: list[str] = []
for f in files:
data = _read_json(f)
if not data:
warnings.append(f"failed to parse {f.name}")
continue
images = data.get("images") or []
anns = data.get("annotations") or []
cats = data.get("categories") or []
ann_count += len(anns)
for c in cats:
name = c.get("name")
if isinstance(name, str):
categories.add(name)
lower = f.name.lower()
if "train" in lower:
split_counts["train"] += len(images)
elif "val" in lower:
split_counts["val"] += len(images)
elif "test" in lower:
split_counts["test"] += len(images)
else:
split_counts["train"] += len(images)
return NormalizedDataset(
format_id=self.format_id,
project=ctx.project,
task=ctx.task,
source_path=str(root),
split_counts=split_counts,
sample_count=sum(split_counts.values()),
annotation_count=ann_count,
artifacts=[self._artifact_name(root, f) for f in files],
warnings=warnings,
extra={"categories": sorted(categories)},
)
@staticmethod
def _artifact_name(root: Path, path: Path) -> str:
try:
return str(path.relative_to(root))
except ValueError:
return path.name

View File

@@ -0,0 +1,67 @@
"""DMS YOLO-style dataset adapter."""
from __future__ import annotations
from pathlib import Path
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".JPG", ".JPEG", ".PNG"}
def _count_images(path: Path) -> int:
if not path.is_dir():
return 0
return sum(1 for p in path.rglob("*") if p.is_file() and p.suffix in IMG_EXTS)
def _count_txt(path: Path) -> int:
if not path.is_dir():
return 0
return sum(1 for p in path.rglob("*.txt") if p.is_file())
class DmsYoloAdapter(IngestAdapter):
format_id = "dms_yolo"
projects = ("dms",)
def can_handle(self, ctx: IngestContext) -> bool:
root = ctx.source_path
return (
(root / "images").is_dir()
and (root / "labels").is_dir()
) or (
(root / "images" / "train").is_dir()
and (root / "labels" / "train").is_dir()
)
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
root = ctx.source_path
train_images = _count_images(root / "images" / "train")
val_images = _count_images(root / "images" / "val")
test_images = _count_images(root / "images" / "test")
if train_images + val_images + test_images == 0:
# fallback single-folder dataset
train_images = _count_images(root / "images")
train_labels = _count_txt(root / "labels" / "train")
val_labels = _count_txt(root / "labels" / "val")
test_labels = _count_txt(root / "labels" / "test")
if train_labels + val_labels + test_labels == 0:
train_labels = _count_txt(root / "labels")
warnings: list[str] = []
if train_images == 0:
warnings.append("train split has no images")
if train_labels == 0:
warnings.append("train split has no labels")
return NormalizedDataset(
format_id=self.format_id,
project=ctx.project,
task=ctx.task,
source_path=str(root),
split_counts={"train": train_images, "val": val_images, "test": test_images},
sample_count=train_images + val_images + test_images,
annotation_count=train_labels + val_labels + test_labels,
artifacts=["images/", "labels/"],
warnings=warnings,
)

View File

@@ -0,0 +1,34 @@
"""Lane .lines.txt adapter."""
from __future__ import annotations
from pathlib import Path
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
class LaneLinesAdapter(IngestAdapter):
format_id = "lane_lines"
projects = ("lane",)
def can_handle(self, ctx: IngestContext) -> bool:
root = ctx.source_path
return any(root.rglob("*.lines.txt"))
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
root = ctx.source_path
line_files = list(root.rglob("*.lines.txt"))
split_counts = {"train": len(line_files), "val": 0, "test": 0}
warnings: list[str] = []
if not line_files:
warnings.append("no *.lines.txt found")
return NormalizedDataset(
format_id=self.format_id,
project=ctx.project,
task=ctx.task,
source_path=str(root),
split_counts=split_counts,
sample_count=len(line_files),
annotation_count=len(line_files),
artifacts=["*.lines.txt"],
warnings=warnings,
)

View File

@@ -0,0 +1,48 @@
"""Lane mask + list txt adapter."""
from __future__ import annotations
from pathlib import Path
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
def _line_count(path: Path) -> int:
if not path.is_file():
return 0
try:
return sum(1 for _ in path.open(encoding="utf-8", errors="ignore"))
except OSError:
return 0
class LaneMaskAdapter(IngestAdapter):
format_id = "lane_mask"
projects = ("lane",)
def can_handle(self, ctx: IngestContext) -> bool:
root = ctx.source_path
return (root / "list" / "train_gt.txt").is_file() or (root / "train_val_gt.txt").is_file()
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
root = ctx.source_path
train = _line_count(root / "list" / "train_gt.txt")
val = _line_count(root / "list" / "val_gt.txt")
test = _line_count(root / "list" / "test_gt.txt")
if train == 0 and (root / "train_val_gt.txt").is_file():
train = _line_count(root / "train_val_gt.txt")
warnings: list[str] = []
if train == 0:
warnings.append("train split list is empty")
return NormalizedDataset(
format_id=self.format_id,
project=ctx.project,
task=ctx.task,
source_path=str(root),
split_counts={"train": train, "val": val, "test": test},
sample_count=train + val + test,
annotation_count=train + val + test,
artifacts=["list/train_gt.txt", "list/val_gt.txt", "list/test_gt.txt"],
warnings=warnings,
)

View File

@@ -0,0 +1,49 @@
"""Adapter registry and auto detection for uploaded datasets."""
from __future__ import annotations
from pathlib import Path
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
from as_platform.data.ingest.dms_coco import DmsCocoAdapter
from as_platform.data.ingest.dms_yolo import DmsYoloAdapter
from as_platform.data.ingest.lane_lines import LaneLinesAdapter
from as_platform.data.ingest.lane_mask import LaneMaskAdapter
class UnknownFormatError(ValueError):
pass
ADAPTERS: tuple[IngestAdapter, ...] = (
DmsYoloAdapter(),
DmsCocoAdapter(),
LaneMaskAdapter(),
LaneLinesAdapter(),
)
def available_formats(project: str) -> list[str]:
return [a.format_id for a in ADAPTERS if project in a.projects]
def detect_adapter(ctx: IngestContext) -> IngestAdapter:
for adapter in ADAPTERS:
if ctx.project not in adapter.projects:
continue
if adapter.can_handle(ctx):
return adapter
raise UnknownFormatError(
f"unable to detect format for project={ctx.project}, task={ctx.task}, "
f"source={ctx.source_path}. supported={available_formats(ctx.project)}"
)
def inspect_uploaded_dataset(project: str, task: str | None, source_path: str | Path) -> NormalizedDataset:
ctx = IngestContext(project=project, task=task, source_path=Path(source_path).resolve())
if not ctx.source_path.exists():
raise FileNotFoundError(f"source path not found: {ctx.source_path}")
adapter = detect_adapter(ctx)
out = adapter.inspect(ctx)
# Ensure adapter id is always reflected in output.
out.format_id = adapter.format_id
return out

View File

@@ -0,0 +1,179 @@
"""Upload candidate lifecycle for data-lake ingestion."""
from __future__ import annotations
import json
import shutil
import tarfile
import uuid
import zipfile
from datetime import datetime
from pathlib import Path
from typing import BinaryIO
from as_platform.config import MANIFESTS
from as_platform.data.catalog_cache import invalidate_catalog_cache
from as_platform.data.ingest import inspect_uploaded_dataset
from as_platform.db.engine import session_scope
from as_platform.db.models import DatasetCandidate
LAKE_ROOT = MANIFESTS / "lake"
UPLOAD_ROOT = LAKE_ROOT / "uploads"
STAGING_ROOT = LAKE_ROOT / "staging"
REPORT_ROOT = LAKE_ROOT / "reports"
def _new_candidate_id() -> str:
return f"cand-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
def _candidate_dirs(candidate_id: str) -> tuple[Path, Path, Path]:
upload_dir = UPLOAD_ROOT / candidate_id
staging_dir = STAGING_ROOT / candidate_id
report_file = REPORT_ROOT / f"{candidate_id}.json"
return upload_dir, staging_dir, report_file
def create_uploaded_candidate(
*,
project: str,
task: str | None,
original_name: str,
upload_size_bytes: int,
submitted_by_name: str | None,
submitted_by_user_id: int | None,
) -> dict:
candidate_id = _new_candidate_id()
upload_dir, _, _ = _candidate_dirs(candidate_id)
upload_dir.mkdir(parents=True, exist_ok=True)
upload_path = upload_dir / original_name
with session_scope() as db:
rec = DatasetCandidate(
id=candidate_id,
project=project,
task=task,
status="uploaded",
source_type="upload",
original_name=original_name,
upload_path=str(upload_path),
upload_size_bytes=upload_size_bytes,
submitted_by_name=submitted_by_name,
submitted_by_user_id=submitted_by_user_id,
)
db.add(rec)
db.flush()
return rec.to_dict()
def write_candidate_upload(candidate_id: str, stream: BinaryIO, chunk_size: int = 1024 * 1024) -> str:
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if not rec:
raise ValueError(f"candidate not found: {candidate_id}")
path = Path(rec.upload_path)
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("wb") as f:
while True:
chunk = stream.read(chunk_size)
if not chunk:
break
f.write(chunk)
rec.upload_size_bytes = path.stat().st_size
db.flush()
return str(path)
def list_candidates(limit: int = 100) -> list[dict]:
with session_scope() as db:
rows = (
db.query(DatasetCandidate)
.order_by(DatasetCandidate.created_at.desc())
.limit(limit)
.all()
)
return [r.to_dict() for r in rows]
def get_candidate(candidate_id: str) -> dict | None:
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
return rec.to_dict() if rec else None
def link_candidate_analysis_job(candidate_id: str, job_id: str) -> None:
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if not rec:
raise ValueError(f"candidate not found: {candidate_id}")
rec.analysis_job_id = job_id
db.flush()
def _extract_to_staging(upload_path: Path, staging_dir: Path) -> Path:
if staging_dir.exists():
shutil.rmtree(staging_dir)
staging_dir.mkdir(parents=True, exist_ok=True)
name = upload_path.name.lower()
if name.endswith(".zip"):
with zipfile.ZipFile(upload_path, "r") as zf:
zf.extractall(staging_dir)
elif name.endswith(".tar") or name.endswith(".tar.gz") or name.endswith(".tgz"):
with tarfile.open(upload_path, "r:*") as tf:
tf.extractall(staging_dir)
else:
raise ValueError(f"unsupported archive format: {upload_path.name}")
subdirs = [p for p in staging_dir.iterdir() if p.is_dir()]
files = [p for p in staging_dir.iterdir() if p.is_file()]
if len(subdirs) == 1 and not files:
return subdirs[0]
return staging_dir
def analyze_uploaded_candidate(candidate_id: str) -> dict:
upload_dir, staging_dir, report_file = _candidate_dirs(candidate_id)
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if not rec:
raise ValueError(f"candidate not found: {candidate_id}")
rec.status = "analyzing"
rec.error_message = None
db.flush()
project = rec.project
task = rec.task
upload_path = Path(rec.upload_path)
if not upload_path.is_file():
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if rec:
rec.status = "failed"
rec.error_message = f"upload file missing: {upload_path}"
raise FileNotFoundError(f"upload file missing: {upload_path}")
try:
dataset_root = _extract_to_staging(upload_path, staging_dir)
normalized = inspect_uploaded_dataset(project, task, dataset_root)
report_file.parent.mkdir(parents=True, exist_ok=True)
report_file.write_text(json.dumps(normalized.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if not rec:
raise ValueError(f"candidate not found during finalize: {candidate_id}")
rec.status = "analyzed"
rec.analyzed_source_path = str(dataset_root)
rec.format_id = normalized.format_id
rec.set_split_counts(normalized.split_counts)
rec.set_quality(normalized.to_dict())
rec.error_message = None
db.flush()
invalidate_catalog_cache()
return normalized.to_dict()
except Exception as e:
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if rec:
rec.status = "failed"
rec.error_message = str(e)
db.flush()
raise

View File

@@ -0,0 +1,38 @@
"""数据整理:校验摘要写入 batch.meta。"""
from __future__ import annotations
from pathlib import Path
from typing import Any
from as_platform.data.batch import META_FILENAME, count_images, count_label_files, read_meta, write_meta
def organize_batch(batch_dir: Path, *, task: str | None = None) -> dict[str, Any]:
"""生成整理报告并合并进 batch.meta.yaml。"""
batch_dir = batch_dir.resolve()
if not batch_dir.is_dir():
raise FileNotFoundError(batch_dir)
images = count_images(batch_dir / "images") + count_images(batch_dir / "images" / "train")
labels = count_label_files(batch_dir / "labels") + count_label_files(batch_dir / "labels" / "train")
report: dict[str, Any] = {
"task": task,
"images": images,
"labels": labels,
"pair_ratio": round(labels / images, 3) if images else 0,
"ready_for_ingest": images > 0 and labels > 0,
"issues": [],
}
if images and not labels:
report["issues"].append("missing_labels")
if labels and not images:
report["issues"].append("missing_images")
meta = read_meta(batch_dir) or {}
meta["organize_report"] = report
meta.setdefault("counts", {})
meta["counts"]["images"] = images
meta["counts"]["labels"] = labels
write_meta(batch_dir, meta)
return report

View File

View File

@@ -0,0 +1,60 @@
"""SQLAlchemy 引擎与会话。"""
from __future__ import annotations
from contextlib import contextmanager
from typing import Generator
from sqlalchemy import create_engine, event, text
from sqlalchemy.orm import Session, declarative_base, sessionmaker
from as_platform.config import DATABASE_URL, IS_POSTGRES, IS_SQLITE
connect_args: dict = {}
engine_kwargs: dict = {"future": True}
if IS_SQLITE:
connect_args["check_same_thread"] = False
elif IS_POSTGRES:
engine_kwargs.update(pool_pre_ping=True, pool_size=5, max_overflow=10)
engine = create_engine(DATABASE_URL, connect_args=connect_args, **engine_kwargs)
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False, future=True)
Base = declarative_base()
@event.listens_for(engine, "connect")
def _on_connect(dbapi_conn, _):
if IS_SQLITE:
cursor = dbapi_conn.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
def check_connection() -> bool:
try:
with engine.connect() as conn:
conn.execute(text("SELECT 1"))
return True
except Exception:
return False
@contextmanager
def session_scope() -> Generator[Session, None, None]:
db = SessionLocal()
try:
yield db
db.commit()
except Exception:
db.rollback()
raise
finally:
db.close()
def get_db() -> Generator[Session, None, None]:
db = SessionLocal()
try:
yield db
finally:
db.close()

View File

@@ -0,0 +1,242 @@
"""数据库初始化、角色权限种子、jsonl 迁移。"""
from __future__ import annotations
import json
from typing import Any
from sqlalchemy import text
from sqlalchemy.orm import Session
from as_platform.config import (
APPROVAL_QUEUE,
FEISHU_ADMIN_DEPARTMENT_IDS,
FEISHU_ADMIN_OPEN_IDS,
IS_POSTGRES,
IS_SQLITE,
JOB_LOG,
MANIFESTS,
)
from as_platform.db.engine import Base, engine, session_scope
from as_platform.db.models import Approval, Job, Permission, Role, User
# 角色 → 权限
ROLE_DEFS: dict[str, tuple[str, list[str]]] = {
"admin": ("管理员", ["*"]),
"reviewer": ("审核员", [
"read:catalog", "read:pending", "read:jobs", "read:audit",
"write:approval_review",
]),
"engineer": ("算法工程师", [
"read:catalog", "read:pending", "read:jobs", "read:audit",
"write:approval_submit",
]),
"labeler": ("标注协调", [
"read:catalog", "read:pending", "write:approval_submit:register",
]),
"viewer": ("只读访客", ["read:catalog", "read:pending"]),
}
PERMISSION_NAMES: dict[str, str] = {
"*": "全部权限",
"read:catalog": "查看数据目录",
"read:pending": "查看送标/批次",
"read:jobs": "查看 Job 队列",
"read:audit": "查看审核记录",
"write:approval_submit": "提交审核(训练/build 等)",
"write:approval_submit:register": "提交批次登记审核",
"write:approval_review": "批准/驳回审核",
"admin:users": "用户与角色管理",
}
def init_database() -> None:
MANIFESTS.mkdir(parents=True, exist_ok=True)
Base.metadata.create_all(bind=engine)
with session_scope() as db:
_ensure_user_columns(db)
_seed_roles_permissions(db)
_import_jsonl_if_empty(db)
_sync_postgres_sequences(db)
def _seed_roles_permissions(db: Session) -> None:
perm_map: dict[str, Permission] = {}
for code, name in PERMISSION_NAMES.items():
p = db.query(Permission).filter_by(code=code).first()
if not p:
p = Permission(code=code, name=name)
db.add(p)
db.flush()
perm_map[code] = p
for role_code, (role_name, perm_codes) in ROLE_DEFS.items():
role = db.query(Role).filter_by(code=role_code).first()
if not role:
role = Role(code=role_code, name=role_name)
db.add(role)
db.flush()
role.permissions = [perm_map[c] for c in perm_codes if c in perm_map]
def _import_jsonl_if_empty(db: Session) -> None:
if db.query(Approval).count() == 0 and APPROVAL_QUEUE.is_file():
for line in APPROVAL_QUEUE.read_text(encoding="utf-8").strip().splitlines():
if not line.strip():
continue
try:
rec = json.loads(line)
except json.JSONDecodeError:
continue
if db.get(Approval, rec.get("id")):
continue
a = Approval(
id=rec["id"],
status=rec.get("status", "pending"),
action=rec["action"],
action_label=rec.get("action_label"),
note=rec.get("note"),
submitted_by_name=rec.get("submitted_by"),
reviewed_by_name=rec.get("reviewed_by"),
review_comment=rec.get("review_comment"),
job_id=rec.get("job_id"),
)
a.set_params(rec.get("params") or {})
if rec.get("result"):
a.set_result(rec["result"])
db.add(a)
if db.query(Job).count() == 0 and JOB_LOG.is_file():
for line in JOB_LOG.read_text(encoding="utf-8").strip().splitlines():
if not line.strip():
continue
try:
rec = json.loads(line)
except json.JSONDecodeError:
continue
if db.get(Job, rec.get("id")):
continue
j = Job(
id=rec["id"],
status=rec.get("status", "queued"),
action=rec["action"],
approval_id=rec.get("approval_id"),
)
j.set_params(rec.get("params") or {})
if rec.get("result"):
j.set_result(rec["result"])
db.add(j)
def assign_default_role(db: Session, user: User) -> None:
"""新用户默认角色;支持 open_id / 部门白名单自动 admin。"""
user_dept_ids = set(user.feishu_department_ids())
if user.feishu_open_id and user.feishu_open_id in FEISHU_ADMIN_OPEN_IDS:
role_code = "admin"
elif user_dept_ids and user_dept_ids.intersection(FEISHU_ADMIN_DEPARTMENT_IDS):
role_code = "admin"
elif not user.roles:
role_code = "engineer"
else:
return
role = db.query(Role).filter_by(code=role_code).first()
if role and role not in user.roles:
user.roles.append(role)
def user_has_permission(user: User, permission: str) -> bool:
if not user or not user.is_active:
return False
for role in user.roles:
for p in role.permissions:
if p.code == "*" or p.code == permission:
return True
# register_batch 专用权限
if permission == "write:approval_submit:register" and p.code == "write:approval_submit":
return True
return False
def user_to_dict(user: User) -> dict[str, Any]:
return {
"id": user.id,
"name": user.name,
"email": user.email,
"avatar_url": user.avatar_url,
"feishu": {
"open_id": user.feishu_open_id,
"union_id": user.feishu_union_id,
"user_id": user.feishu_user_id,
"tenant_key": user.feishu_tenant_key,
"department_ids": user.feishu_department_ids(),
},
"roles": [{"code": r.code, "name": r.name} for r in user.roles],
"permissions": _collect_permissions(user),
}
def _collect_permissions(user: User) -> list[str]:
codes: set[str] = set()
for role in user.roles:
for p in role.permissions:
codes.add(p.code)
return sorted(codes)
def _sync_postgres_sequences(db: Session) -> None:
"""修复 PostgreSQL 自增序列与现有数据不一致问题。"""
if not IS_POSTGRES:
return
# 当历史迁移手动插入了 idsequence 可能还停留在 1导致 duplicate key。
db.execute(
text(
"""
SELECT setval(
pg_get_serial_sequence('users', 'id'),
COALESCE((SELECT MAX(id) FROM users), 1),
true
)
"""
)
)
db.execute(
text(
"""
SELECT setval(
pg_get_serial_sequence('roles', 'id'),
COALESCE((SELECT MAX(id) FROM roles), 1),
true
)
"""
)
)
db.execute(
text(
"""
SELECT setval(
pg_get_serial_sequence('permissions', 'id'),
COALESCE((SELECT MAX(id) FROM permissions), 1),
true
)
"""
)
)
def _ensure_user_columns(db: Session) -> None:
user_columns = {
"feishu_user_id": "VARCHAR(64)",
"feishu_tenant_key": "VARCHAR(128)",
"feishu_department_ids_json": "TEXT",
}
if IS_POSTGRES:
for column, column_type in user_columns.items():
db.execute(text(f"ALTER TABLE users ADD COLUMN IF NOT EXISTS {column} {column_type}"))
db.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS ix_users_feishu_user_id ON users (feishu_user_id)"))
return
if IS_SQLITE:
rows = db.execute(text("PRAGMA table_info(users)")).fetchall()
existing = {str(row[1]) for row in rows}
for column, column_type in user_columns.items():
if column not in existing:
db.execute(text(f"ALTER TABLE users ADD COLUMN {column} {column_type}"))

View File

@@ -0,0 +1,243 @@
"""ORM 模型。"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from sqlalchemy import (
Boolean,
Column,
DateTime,
ForeignKey,
Integer,
String,
Table,
Text,
)
from sqlalchemy.orm import relationship
from as_platform.db.engine import Base
user_roles = Table(
"user_roles",
Base.metadata,
Column("user_id", Integer, ForeignKey("users.id", ondelete="CASCADE"), primary_key=True),
Column("role_id", Integer, ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True),
)
role_permissions = Table(
"role_permissions",
Base.metadata,
Column("role_id", Integer, ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True),
Column("permission_id", Integer, ForeignKey("permissions.id", ondelete="CASCADE"), primary_key=True),
)
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
class User(Base):
__tablename__ = "users"
id = Column(Integer, primary_key=True)
feishu_open_id = Column(String(64), unique=True, nullable=True, index=True)
feishu_union_id = Column(String(64), unique=True, nullable=True, index=True)
feishu_user_id = Column(String(64), unique=True, nullable=True, index=True)
feishu_tenant_key = Column(String(128), nullable=True)
feishu_department_ids_json = Column(Text, nullable=True)
name = Column(String(128), nullable=False, default="")
email = Column(String(256), nullable=True)
avatar_url = Column(String(512), nullable=True)
is_active = Column(Boolean, default=True, nullable=False)
created_at = Column(DateTime(timezone=True), default=_utcnow)
updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
roles = relationship("Role", secondary=user_roles, back_populates="users")
def feishu_department_ids(self) -> list[str]:
if not self.feishu_department_ids_json:
return []
try:
data = json.loads(self.feishu_department_ids_json)
if isinstance(data, list):
return [str(x) for x in data]
except Exception:
return []
return []
class Role(Base):
__tablename__ = "roles"
id = Column(Integer, primary_key=True)
code = Column(String(32), unique=True, nullable=False)
name = Column(String(64), nullable=False)
users = relationship("User", secondary=user_roles, back_populates="roles")
permissions = relationship("Permission", secondary=role_permissions, back_populates="roles")
class Permission(Base):
__tablename__ = "permissions"
id = Column(Integer, primary_key=True)
code = Column(String(64), unique=True, nullable=False)
name = Column(String(128), nullable=False)
roles = relationship("Role", secondary=role_permissions, back_populates="permissions")
class Approval(Base):
__tablename__ = "approvals"
id = Column(String(64), primary_key=True)
status = Column(String(32), nullable=False, default="pending", index=True)
action = Column(String(64), nullable=False)
action_label = Column(String(128), nullable=True)
params_json = Column(Text, nullable=False, default="{}")
note = Column(Text, nullable=True)
submitted_by_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
submitted_by_name = Column(String(128), nullable=True)
submitted_at = Column(DateTime(timezone=True), default=_utcnow)
reviewed_by_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
reviewed_by_name = Column(String(128), nullable=True)
reviewed_at = Column(DateTime(timezone=True), nullable=True)
review_comment = Column(Text, nullable=True)
job_id = Column(String(64), nullable=True)
executed_at = Column(DateTime(timezone=True), nullable=True)
result_json = Column(Text, nullable=True)
def params(self) -> dict:
return json.loads(self.params_json or "{}")
def set_params(self, data: dict) -> None:
self.params_json = json.dumps(data, ensure_ascii=False)
def result(self) -> dict | None:
if not self.result_json:
return None
return json.loads(self.result_json)
def set_result(self, data: dict) -> None:
self.result_json = json.dumps(data, ensure_ascii=False)
def to_dict(self) -> dict:
return {
"id": self.id,
"status": self.status,
"action": self.action,
"action_label": self.action_label,
"params": self.params(),
"note": self.note,
"submitted_by": self.submitted_by_name,
"submitted_by_user_id": self.submitted_by_user_id,
"submitted_at": self.submitted_at.isoformat() if self.submitted_at else None,
"reviewed_by": self.reviewed_by_name,
"reviewed_by_user_id": self.reviewed_by_user_id,
"reviewed_at": self.reviewed_at.isoformat() if self.reviewed_at else None,
"review_comment": self.review_comment,
"job_id": self.job_id,
"executed_at": self.executed_at.isoformat() if self.executed_at else None,
"result": self.result(),
}
class Job(Base):
__tablename__ = "jobs"
id = Column(String(64), primary_key=True)
status = Column(String(32), nullable=False, default="queued", index=True)
action = Column(String(64), nullable=False)
params_json = Column(Text, nullable=False, default="{}")
approval_id = Column(String(64), ForeignKey("approvals.id"), nullable=True)
created_at = Column(DateTime(timezone=True), default=_utcnow)
started_at = Column(DateTime(timezone=True), nullable=True)
finished_at = Column(DateTime(timezone=True), nullable=True)
result_json = Column(Text, nullable=True)
def params(self) -> dict:
return json.loads(self.params_json or "{}")
def set_params(self, data: dict) -> None:
self.params_json = json.dumps(data, ensure_ascii=False)
def result(self) -> dict | None:
if not self.result_json:
return None
return json.loads(self.result_json)
def set_result(self, data: dict) -> None:
self.result_json = json.dumps(data, ensure_ascii=False)
def to_dict(self) -> dict:
return {
"id": self.id,
"status": self.status,
"action": self.action,
"params": self.params(),
"approval_id": self.approval_id,
"created_at": self.created_at.isoformat() if self.created_at else None,
"started_at": self.started_at.isoformat() if self.started_at else None,
"finished_at": self.finished_at.isoformat() if self.finished_at else None,
"result": self.result(),
}
class DatasetCandidate(Base):
__tablename__ = "dataset_candidates"
id = Column(String(64), primary_key=True)
project = Column(String(32), nullable=False, index=True)
task = Column(String(64), nullable=True, index=True)
status = Column(String(32), nullable=False, default="uploaded", index=True)
source_type = Column(String(32), nullable=False, default="upload")
original_name = Column(String(255), nullable=True)
upload_path = Column(String(1024), nullable=False)
analyzed_source_path = Column(String(1024), nullable=True)
format_id = Column(String(64), nullable=True)
split_counts_json = Column(Text, nullable=False, default="{}")
quality_json = Column(Text, nullable=True)
error_message = Column(Text, nullable=True)
upload_size_bytes = Column(Integer, nullable=False, default=0)
submitted_by_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
submitted_by_name = Column(String(128), nullable=True)
analysis_job_id = Column(String(64), ForeignKey("jobs.id"), nullable=True)
created_at = Column(DateTime(timezone=True), default=_utcnow)
updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
def split_counts(self) -> dict:
return json.loads(self.split_counts_json or "{}")
def set_split_counts(self, data: dict) -> None:
self.split_counts_json = json.dumps(data, ensure_ascii=False)
def quality(self) -> dict | None:
if not self.quality_json:
return None
return json.loads(self.quality_json)
def set_quality(self, data: dict) -> None:
self.quality_json = json.dumps(data, ensure_ascii=False)
def to_dict(self) -> dict:
return {
"id": self.id,
"project": self.project,
"task": self.task,
"status": self.status,
"source_type": self.source_type,
"original_name": self.original_name,
"upload_path": self.upload_path,
"analyzed_source_path": self.analyzed_source_path,
"format_id": self.format_id,
"split_counts": self.split_counts(),
"quality": self.quality(),
"error_message": self.error_message,
"upload_size_bytes": self.upload_size_bytes,
"submitted_by_user_id": self.submitted_by_user_id,
"submitted_by_name": self.submitted_by_name,
"analysis_job_id": self.analysis_job_id,
"created_at": self.created_at.isoformat() if self.created_at else None,
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
}

View File

View File

@@ -0,0 +1,145 @@
"""Job 队列PostgreSQL + 可选 Redis Worker"""
from __future__ import annotations
import threading
import uuid
from datetime import datetime, timezone
from typing import Any
from as_platform.config import JOB_EXECUTOR
from as_platform.db.engine import session_scope
from as_platform.db.models import Job
_executor_lock = threading.Lock()
def _now() -> datetime:
return datetime.now(timezone.utc)
def _new_id() -> str:
return f"job-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
def enqueue_job(
action: str,
params: dict[str, Any],
*,
approval_id: str | None = None,
async_run: bool = True,
) -> dict[str, Any]:
job_id = _new_id()
with session_scope() as db:
job = Job(
id=job_id,
status="queued",
action=action,
approval_id=approval_id,
created_at=_now(),
)
job.set_params(params)
db.add(job)
out = get_job(job_id) or {"id": job_id, "status": "queued", "action": action}
if not async_run:
_run_job(job_id)
return get_job(job_id) or out
if JOB_EXECUTOR == "worker":
from as_platform.redis.bus import push_job
push_job(job_id)
return out
threading.Thread(target=_run_job, args=(job_id,), daemon=True).start()
return out
def get_job(job_id: str) -> dict[str, Any] | None:
with session_scope() as db:
rec = db.get(Job, job_id)
return rec.to_dict() if rec else None
def list_jobs(status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
with session_scope() as db:
q = db.query(Job).order_by(Job.created_at.desc())
if status:
q = q.filter(Job.status == status)
return [j.to_dict() for j in q.limit(limit).all()]
def _patch(job_id: str, **fields: Any) -> dict[str, Any] | None:
with session_scope() as db:
rec = db.get(Job, job_id)
if not rec:
return None
for k, v in fields.items():
if k == "result" and isinstance(v, dict):
rec.set_result(v)
elif hasattr(rec, k):
setattr(rec, k, v)
db.flush()
return rec.to_dict()
def _compact_result(payload: Any) -> dict[str, Any]:
if isinstance(payload, dict):
out = dict(payload)
else:
out = {"value": payload}
if "ok" not in out:
out["ok"] = True
for k in ("stdout", "stderr"):
if isinstance(out.get(k), str):
out[k] = out[k][-8000:]
return out
def _run_job(job_id: str) -> None:
with _executor_lock:
job = get_job(job_id)
if not job or job.get("status") not in ("queued",):
return
_patch(job_id, status="running", started_at=_now())
from as_platform.agents.trace import trace_span
from as_platform.jobs.runner import execute_action
from as_platform.redis.bus import publish
publish("job.started", {"job_id": job_id, "action": job["action"]})
try:
with trace_span("job_start", job_id=job_id, action=job["action"], approval_id=job.get("approval_id")):
result = execute_action(job["action"], job.get("params") or {})
persisted = _compact_result(result)
_patch(
job_id,
status="succeeded",
finished_at=_now(),
result=persisted,
)
publish("job.succeeded", {"job_id": job_id})
with trace_span("job_end", job_id=job_id, status="succeeded"):
pass
_sync_approval(job.get("approval_id"), "executed", persisted)
except Exception as e:
_patch(job_id, status="failed", finished_at=_now(), result={"ok": False, "error": str(e)})
publish("job.failed", {"job_id": job_id, "error": str(e)})
with trace_span("job_end", job_id=job_id, status="failed", error=str(e)):
pass
_sync_approval(job.get("approval_id"), "failed", {"error": str(e)})
def _sync_approval(approval_id: str | None, status: str, result: dict) -> None:
if not approval_id:
return
from as_platform.audit.queue import _update, _now as audit_now
_update(
approval_id,
status=status,
executed_at=audit_now(),
result=result if isinstance(result, dict) and "ok" in result else {"ok": status == "executed", **result},
)

View File

@@ -0,0 +1,159 @@
"""执行动作优先引擎适配器fallback as.py CLI。"""
from __future__ import annotations
import json
import subprocess
import sys
from typing import Any
from as_platform.config import WORKSPACE, PLATFORM_DIR, LANE_DATA_VIZ_ENABLED
if str(WORKSPACE) not in sys.path:
sys.path.insert(0, str(WORKSPACE))
if str(PLATFORM_DIR) not in sys.path:
sys.path.insert(0, str(PLATFORM_DIR))
ML_PY = WORKSPACE / "as.py"
AS_PY = ML_PY
LONG_ACTIONS = {"train_dms", "train_lane", "pipeline_dms", "eval_dms", "eval_lane", "visualize_dms", "visualize_lane"}
def _run_ml(argv: list[str], timeout: int = 7200) -> dict[str, Any]:
cmd = [sys.executable, str(ML_PY), *argv]
proc = subprocess.run(cmd, cwd=str(WORKSPACE), capture_output=True, text=True, timeout=timeout)
if proc.returncode != 0:
raise RuntimeError(f"as.py 失败 (exit {proc.returncode}):\n{proc.stderr or proc.stdout}")
return {"ok": True, "stdout": proc.stdout, "stderr": proc.stderr, "command": " ".join(cmd)}
def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]:
p = params or {}
if action == "train_dms":
track = p.get("track", "platform")
if track == "local":
from algorithms.dms_yolo.adapter import train_local
return train_local(p["task"], p.get("mode", "full"), p.get("config_overrides"))
from algorithms.dms_yolo.adapter import train_platform
return train_platform(p["task"], p.get("mode", "full"))
if action == "train_lane":
track = p.get("track", "platform")
if track == "local":
from algorithms.lane_ufld.adapter import train_local
return train_local(p.get("config_overrides"))
from algorithms.lane_ufld.adapter import train_platform
return train_platform()
if action == "train_dms_legacy":
argv = ["train", "dms", p["task"]]
if p.get("mode"):
argv.extend(["--mode", str(p["mode"])])
return _run_ml(argv, timeout=86400)
if action == "train_lane_legacy":
return _run_ml(["train", "lane"], timeout=86400)
if action == "build_dms":
argv = ["build", "dms", p["task"]]
if p.get("pack"):
argv.extend(["--pack", str(p["pack"])])
if p.get("batch"):
argv.extend(["--batch", str(p["batch"])])
if p.get("all_sources"):
argv.append("--all-sources")
if p.get("dry_run"):
argv.append("--dry-run")
if p.get("skip_validate"):
argv.append("--skip-validate")
if p.get("no_refresh"):
argv.append("--no-refresh")
return _run_ml(argv)
if action == "build_lane":
return _run_ml(["build", "lane"])
if action == "enable_pack":
return _run_ml(["enable", p["project"], p["pack"]])
if action == "disable_pack":
return _run_ml(["disable", p["project"], p["pack"]])
if action == "eval_dms":
argv = ["eval", "dms", p["task"]]
if p.get("save_candidate"):
argv.append("--save-candidate")
if p.get("weights"):
argv.extend(["--weights", str(p["weights"])])
return _run_ml(argv, timeout=3600)
if action == "eval_lane":
from algorithms.lane_ufld.adapter import eval_task
return eval_task(
model_path=p.get("model_path"),
data_root=p.get("data_root"),
test_list=p.get("test_list", "list/test_gt.txt"),
)
if action == "visualize_dms":
from algorithms.dms_yolo.adapter import visualize_task
return visualize_task(
p["task"],
weights=p.get("weights"),
)
if action == "visualize_lane":
if not LANE_DATA_VIZ_ENABLED:
raise RuntimeError("车道线数据可视化暂未开放")
from algorithms.lane_ufld.adapter import visualize_task
return visualize_task(
model_path=p.get("model_path"),
data_root=p.get("data_root"),
test_list=p.get("test_list", "list/test_gt.txt"),
)
if action == "promote_dms":
argv = ["promote", "dms", p["task"]]
if p.get("force"):
argv.append("--force")
return _run_ml(argv)
if action == "pipeline_dms":
argv = ["pipeline", "dms", p["task"], "--pack", str(p.get("pack", "dms_v2"))]
if p.get("batch"):
argv.extend(["--batch", str(p["batch"])])
if p.get("all_sources"):
argv.append("--all-sources")
if p.get("train"):
argv.append("--train")
if p.get("dry_run"):
argv.append("--dry-run")
return _run_ml(argv, timeout=86400)
if action == "register_batch":
from as_platform.data.core import register_batch
register_batch(
None, p["project"], p.get("task"), p["batch"],
pack=p.get("pack"), stage=p.get("stage", "returned"),
engineer=p.get("engineer"), location=p.get("location", "inbox"),
)
return {"ok": True, "stdout": "register_batch ok", "stderr": ""}
if action == "analyze_uploaded_dataset":
from as_platform.data.lake import analyze_uploaded_candidate
candidate_id = p["candidate_id"]
result = analyze_uploaded_candidate(candidate_id)
return {
"ok": True,
"stdout": json.dumps(result, ensure_ascii=False),
"stderr": "",
"result": result,
}
raise ValueError(f"未实现执行: {action}")

View File

View File

@@ -0,0 +1,51 @@
"""Redis 连接与 Job 事件总线。"""
from __future__ import annotations
import json
from functools import lru_cache
from typing import Any
from as_platform.config import JOB_QUEUE_KEY, REDIS_URL
try:
import redis
except ImportError:
redis = None # type: ignore
@lru_cache(maxsize=1)
def get_redis():
if not REDIS_URL or redis is None:
return None
return redis.from_url(REDIS_URL, decode_responses=True)
def ping_redis() -> bool:
try:
r = get_redis()
return bool(r and r.ping())
except Exception:
return False
def publish(event: str, payload: dict[str, Any]) -> None:
r = get_redis()
if not r:
return
r.publish("as:events", json.dumps({"event": event, **payload}, ensure_ascii=False))
def push_job(job_id: str) -> None:
r = get_redis()
if not r:
raise RuntimeError("Redis 未配置,无法使用 worker 模式")
r.lpush(JOB_QUEUE_KEY, job_id)
publish("job.queued", {"job_id": job_id})
def pop_job(timeout: int = 5) -> str | None:
r = get_redis()
if not r:
return None
item = r.brpop(JOB_QUEUE_KEY, timeout=timeout)
return item[1] if item else None

View File

@@ -0,0 +1,37 @@
"""华胥平台 Python SDK — CLI / Job / Agent 共用。"""
from __future__ import annotations
from as_platform.audit.queue import (
approve_and_execute,
get_approval,
list_approvals,
reject_approval,
submit_approval,
)
from as_platform.config import WORKSPACE
from as_platform.data.core import get_catalog, get_pending_report, load_wf, register_batch
from as_platform.data.organize import organize_batch
from as_platform.jobs.queue import enqueue_job, get_job, list_jobs
from as_platform.agents.tools import invoke_tool, TOOL_REGISTRY
from as_platform.agents.trace import get_trace, start_trace
__all__ = [
"WORKSPACE",
"get_pending_report",
"get_catalog",
"register_batch",
"organize_batch",
"load_wf",
"submit_approval",
"list_approvals",
"get_approval",
"approve_and_execute",
"reject_approval",
"enqueue_job",
"get_job",
"list_jobs",
"invoke_tool",
"TOOL_REGISTRY",
"get_trace",
"start_trace",
]

View File

@@ -0,0 +1 @@
"""训练记录聚合Job + Approval + 模型 manifest"""

View File

@@ -0,0 +1,216 @@
"""训练记录查询与提交。"""
from __future__ import annotations
import json
from datetime import datetime
from pathlib import Path
from typing import Any
import yaml
from as_platform.audit.queue import ACTION_LABELS, get_approval, submit_approval
from as_platform.config import WORKSPACE
from as_platform.jobs.queue import get_job, list_jobs
TRAINING_ACTIONS = frozenset(
{
"train_dms",
"train_lane",
"eval_dms",
"eval_lane",
"promote_dms",
"pipeline_dms",
"visualize_dms",
"visualize_lane",
}
)
ACTION_KIND = {
"train_dms": "train",
"train_lane": "train",
"eval_dms": "eval",
"eval_lane": "eval",
"promote_dms": "promote",
"pipeline_dms": "pipeline",
"visualize_dms": "visualize",
"visualize_lane": "visualize",
}
def _project_for_action(action: str) -> str:
if action.endswith("_dms") or action.startswith("promote_dms") or action.startswith("pipeline_dms"):
return "dms"
if "lane" in action:
return "lane"
return "unknown"
def _parse_ts(ts: str | None) -> datetime | None:
if not ts:
return None
try:
return datetime.fromisoformat(ts.replace("Z", "+00:00"))
except ValueError:
return None
def _duration_sec(job: dict[str, Any]) -> float | None:
start = _parse_ts(job.get("started_at") or job.get("created_at"))
end = _parse_ts(job.get("finished_at"))
if not start or not end:
return None
return max(0.0, (end - start).total_seconds())
def _extract_weight(job: dict[str, Any]) -> str | None:
params = job.get("params") or {}
result = job.get("result") or {}
for key in ("best_weights", "candidate", "model_path", "weights", "run_dir"):
val = result.get(key) or params.get(key)
if isinstance(val, str) and val.strip():
return val.strip()
return None
def _extract_metrics(result: dict[str, Any]) -> dict[str, Any]:
if not isinstance(result, dict):
return {}
metrics: dict[str, Any] = {}
for key in ("map50", "map50_95", "map", "delta_map50", "precision", "recall", "f1"):
if key in result and result[key] is not None:
metrics[key] = result[key]
if "metrics" in result and isinstance(result["metrics"], dict):
metrics.update(result["metrics"])
if "last_eval" in result and isinstance(result["last_eval"], dict):
metrics.update(result["last_eval"])
return metrics
def enrich_job(job: dict[str, Any]) -> dict[str, Any]:
action = job.get("action", "")
params = job.get("params") or {}
result = job.get("result") or {}
approval = get_approval(job["approval_id"]) if job.get("approval_id") else None
task = params.get("task")
if action == "train_lane" and not task:
task = None
return {
**job,
"action_label": ACTION_LABELS.get(action, action),
"project": _project_for_action(action),
"kind": ACTION_KIND.get(action, "other"),
"task": task,
"track": params.get("track"),
"weight_path": _extract_weight(job),
"metrics": _extract_metrics(result),
"error": result.get("error") if isinstance(result, dict) else None,
"approval": approval,
"duration_sec": _duration_sec(job),
}
def _summarize(records: list[dict[str, Any]]) -> dict[str, int]:
summary = {"total": len(records), "running": 0, "queued": 0, "succeeded": 0, "failed": 0}
for rec in records:
status = rec.get("status") or ""
if status in summary:
summary[status] += 1
return summary
def list_training_records(
*,
project: str | None = None,
kind: str | None = None,
status: str | None = None,
task: str | None = None,
limit: int = 100,
) -> dict[str, Any]:
jobs = list_jobs(status=status, limit=500)
records: list[dict[str, Any]] = []
for job in jobs:
if job.get("action") not in TRAINING_ACTIONS:
continue
rec = enrich_job(job)
if project and rec["project"] != project:
continue
if kind and rec["kind"] != kind:
continue
if task and rec.get("task") != task:
continue
records.append(rec)
if len(records) >= limit:
break
return {"items": records, "total": len(records), "summary": _summarize(records)}
def get_training_record(job_id: str) -> dict[str, Any] | None:
job = get_job(job_id)
if not job or job.get("action") not in TRAINING_ACTIONS:
return None
return enrich_job(job)
def _read_train_versions() -> dict[str, Any]:
path = WORKSPACE / "datasets/dms/manifests/train_versions.yaml"
if not path.is_file():
return {}
data = yaml.safe_load(path.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else {}
def _read_eval_log(task: str | None = None, limit: int = 30) -> list[dict[str, Any]]:
path = WORKSPACE / "datasets/dms/manifests/eval_log.jsonl"
if not path.is_file():
return []
lines = path.read_text(encoding="utf-8").splitlines()
entries: list[dict[str, Any]] = []
for line in reversed(lines):
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except json.JSONDecodeError:
continue
if task and row.get("task") != task:
continue
entries.append(row)
if len(entries) >= limit:
break
return entries
def get_model_registry(project: str = "dms", task: str | None = None) -> dict[str, Any]:
if project != "dms":
return {"project": project, "tasks": {}, "eval_history": []}
versions = _read_train_versions()
if task:
task_data = versions.get(task, {})
return {
"project": "dms",
"task": task,
"version": task_data,
"eval_history": _read_eval_log(task=task),
}
tasks = {name: data for name, data in versions.items() if isinstance(data, dict)}
return {"project": "dms", "tasks": tasks, "eval_history": _read_eval_log(limit=20)}
def create_training_submission(
action: str,
params: dict[str, Any],
*,
submitted_by: str | None = None,
submitted_by_user_id: int | None = None,
note: str | None = None,
) -> dict[str, Any]:
if action not in TRAINING_ACTIONS:
raise ValueError(f"不支持的动作: {action}")
return submit_approval(
action,
params,
submitted_by=submitted_by,
submitted_by_user_id=submitted_by_user_id,
note=note,
)

3
platform/web/env.sh Normal file
View File

@@ -0,0 +1,3 @@
#!/usr/bin/env bash
# 将 Node.js~/.local/node加入 PATH供 npm run dev 使用
export PATH="${HOME}/.local/node/bin:${PATH}"

15
platform/web/index.html Normal file
View File

@@ -0,0 +1,15 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>HSAP · Huaxu Sentinel Active Safety Platform</title>
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link href="https://fonts.googleapis.com/css2?family=DM+Sans:opsz,wght@9..40,400;9..40,500;9..40,600;9..40,700&family=Noto+Sans+SC:wght@400;500;600;700&display=swap" rel="stylesheet" />
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>

1878
platform/web/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

23
platform/web/package.json Normal file
View File

@@ -0,0 +1,23 @@
{
"name": "huaxu-as-web",
"private": true,
"version": "1.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "tsc -b && vite build",
"preview": "vite preview"
},
"dependencies": {
"react": "^19.0.0",
"react-dom": "^19.0.0",
"react-router-dom": "^7.1.0"
},
"devDependencies": {
"@types/react": "^19.0.0",
"@types/react-dom": "^19.0.0",
"@vitejs/plugin-react": "^4.3.4",
"typescript": "~5.7.2",
"vite": "^6.0.0"
}
}

47
platform/web/src/App.tsx Normal file
View File

@@ -0,0 +1,47 @@
import { BrowserRouter, Navigate, Route, Routes } from "react-router-dom";
import { AuthProvider } from "./auth/AuthContext";
import { RequireAuth } from "./auth/RequireAuth";
import { Layout } from "./components/Layout";
import { ToastProvider } from "./components/Toast";
import { AuthCallbackPage } from "./pages/AuthCallback";
import { LoginPage } from "./pages/Login";
import { LabelingPage } from "./pages/Labeling";
import { CatalogPage } from "./pages/Catalog";
import { AuditDetailPage, AuditPage } from "./pages/Audit";
import { JobsPage } from "./pages/Jobs";
import { TrainingPage } from "./pages/Training";
import { LogsPage } from "./pages/Logs";
export default function App() {
return (
<ToastProvider>
<AuthProvider>
<BrowserRouter>
<Routes>
<Route path="/login" element={<LoginPage />} />
<Route path="/auth/callback" element={<AuthCallbackPage />} />
<Route
path="/"
element={
<RequireAuth>
<Layout />
</RequireAuth>
}
>
<Route index element={<Navigate to="/labeling" replace />} />
<Route path="labeling" element={<LabelingPage />} />
<Route path="catalog" element={<CatalogPage />} />
<Route path="audit" element={<AuditPage />} />
<Route path="audit/:id" element={<AuditDetailPage />} />
<Route path="jobs" element={<JobsPage />} />
<Route path="uploads" element={<Navigate to="/labeling" replace />} />
<Route path="training" element={<TrainingPage />} />
<Route path="iterate" element={<Navigate to="/training" replace />} />
<Route path="logs" element={<LogsPage />} />
</Route>
</Routes>
</BrowserRouter>
</AuthProvider>
</ToastProvider>
);
}

View File

@@ -0,0 +1,378 @@
const API_BASE = import.meta.env.VITE_API_BASE || "";
let _token: string | null = localStorage.getItem("as_access_token");
function authHeaders(): Record<string, string> {
const h: Record<string, string> = { "Content-Type": "application/json" };
if (_token) h.Authorization = `Bearer ${_token}`;
return h;
}
async function fetchJson<T>(url: string, init?: RequestInit): Promise<T> {
const res = await fetch(url, {
...init,
cache: "no-store",
headers: { ...authHeaders(), ...(init?.headers as Record<string, string>) },
});
if (res.status === 401) throw new Error("UNAUTHORIZED");
if (!res.ok) throw new Error(await res.text() || res.statusText);
return (await res.json()) as T;
}
async function postJson<T = unknown>(url: string, body?: unknown): Promise<T> {
return fetchJson<T>(url, {
method: "POST",
body: body !== undefined ? JSON.stringify(body) : undefined,
});
}
function uploadWithProgress(
url: string,
formData: FormData,
onProgress?: (percent: number) => void
): Promise<{ candidate: DataCandidate; job: JobRecord }> {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("POST", url, true);
if (_token) xhr.setRequestHeader("Authorization", `Bearer ${_token}`);
xhr.upload.onprogress = (evt) => {
if (!onProgress || !evt.lengthComputable) return;
const pct = Math.round((evt.loaded / evt.total) * 100);
onProgress(pct);
};
xhr.onerror = () => reject(new Error("上传失败"));
xhr.onload = () => {
if (xhr.status === 401) return reject(new Error("UNAUTHORIZED"));
if (xhr.status < 200 || xhr.status >= 300) return reject(new Error(xhr.responseText || `HTTP ${xhr.status}`));
try {
const parsed = JSON.parse(xhr.responseText || "{}");
resolve(parsed);
} catch (e) {
reject(new Error("上传响应解析失败"));
}
};
xhr.send(formData);
});
}
export type AuthUser = {
id: number;
name: string;
email?: string;
avatar_url?: string;
roles: { code: string; name: string }[];
permissions: string[];
};
export const api = {
base: API_BASE || window.location.origin,
setToken(token: string | null) {
_token = token;
},
authConfig: () => fetchJson<{ feishu_enabled: boolean; dev_auth_enabled: boolean }>(`${API_BASE}/api/v1/auth/config`),
me: () => fetchJson<AuthUser>(`${API_BASE}/api/v1/auth/me`),
devLogin: (name?: string) => postJson<{ access_token: string; user: AuthUser }>(`${API_BASE}/api/v1/auth/dev/login`, { name: name || "开发用户" }),
health: () =>
fetchJson<{
status: string;
workspace?: string;
database?: string;
db_connected?: string;
redis_connected?: string;
}>(`${API_BASE}/api/v1/health`),
pending: () => fetchJson<PendingReport>(`${API_BASE}/api/v1/pending`),
catalog: (refresh = false) =>
fetchJson<CatalogReport>(`${API_BASE}/api/v1/catalog${refresh ? "?refresh=true" : ""}`),
listApprovals: (status?: string) => {
const q = status ? `?status=${encodeURIComponent(status)}` : "";
return fetchJson<{ items: ApprovalRecord[] }>(`${API_BASE}/api/v1/approvals${q}`);
},
getApproval: (id: string) => fetchJson<ApprovalRecord>(`${API_BASE}/api/v1/approvals/${encodeURIComponent(id)}`),
getApprovalPreview: (id: string) =>
fetchJson<AuditPreview>(`${API_BASE}/api/v1/approvals/${encodeURIComponent(id)}/preview`),
listApprovalImages: (id: string, offset = 0, limit = 60) =>
fetchJson<{ total: number; offset: number; limit: number; items: AuditImageItem[] }>(
`${API_BASE}/api/v1/approvals/${encodeURIComponent(id)}/images?offset=${offset}&limit=${limit}`
),
fetchApprovalImageBlob: async (approvalId: string, imageId: string, thumb = true): Promise<string> => {
const q = thumb ? "?thumb=true" : "?thumb=false";
const res = await fetch(
`${API_BASE}/api/v1/approvals/${encodeURIComponent(approvalId)}/images/${encodeURIComponent(imageId)}${q}`,
{ headers: authHeaders(), cache: "no-store" }
);
if (!res.ok) throw new Error(await res.text() || res.statusText);
return URL.createObjectURL(await res.blob());
},
submitApproval: (action: string, params: Record<string, unknown>, note?: string) =>
postJson(`${API_BASE}/api/v1/approvals/submit`, { action, params, note }),
submitBuildBatch: (body: Record<string, unknown>) =>
postJson(`${API_BASE}/api/v1/approvals/submit-build-batch`, body),
approve: (id: string, comment?: string) =>
postJson(`${API_BASE}/api/v1/approvals/${id}/approve`, { comment }),
reject: (id: string, comment?: string) =>
postJson(`${API_BASE}/api/v1/approvals/${id}/reject`, { comment }),
registerBatch: (body: Record<string, unknown>) =>
postJson(`${API_BASE}/api/v1/register-batch`, body),
listJobs: (status?: string) => {
const q = status ? `?status=${encodeURIComponent(status)}` : "";
return fetchJson<{ items: JobRecord[] }>(`${API_BASE}/api/v1/jobs${q}`).catch(() => ({ items: [] }));
},
listTrainingRecords: (opts?: {
project?: string;
kind?: string;
status?: string;
task?: string;
limit?: number;
}) => {
const params = new URLSearchParams();
if (opts?.project) params.set("project", opts.project);
if (opts?.kind) params.set("kind", opts.kind);
if (opts?.status) params.set("status", opts.status);
if (opts?.task) params.set("task", opts.task);
if (opts?.limit) params.set("limit", String(opts.limit));
const q = params.toString();
return fetchJson<{ items: TrainingRecord[]; total: number; summary: TrainingSummary }>(
`${API_BASE}/api/v1/training/records${q ? `?${q}` : ""}`
);
},
getTrainingRecord: (jobId: string) =>
fetchJson<TrainingRecord>(`${API_BASE}/api/v1/training/records/${encodeURIComponent(jobId)}`),
getModelRegistry: (project = "dms", task?: string) => {
const params = new URLSearchParams({ project });
if (task) params.set("task", task);
return fetchJson<ModelRegistry>(`${API_BASE}/api/v1/training/models?${params.toString()}`);
},
createTrainingRecord: (action: string, params: Record<string, unknown>, note?: string) =>
postJson<ApprovalRecord>(`${API_BASE}/api/v1/training/records`, { action, params, note }),
uploadDatasetFile: (
file: File,
project: string,
task?: string,
onProgress?: (percent: number) => void
) => {
const formData = new FormData();
formData.append("project", project);
if (task) formData.append("task", task);
formData.append("file", file);
return uploadWithProgress(`${API_BASE}/api/v1/data/upload/file`, formData, onProgress);
},
listDataCandidates: (limit = 50) =>
fetchJson<{ items: DataCandidate[] }>(`${API_BASE}/api/v1/data/candidates?limit=${limit}`),
getDataCandidate: (candidateId: string) =>
fetchJson<DataCandidate>(`${API_BASE}/api/v1/data/candidates/${encodeURIComponent(candidateId)}`),
inspectUploadPath: (project: "dms" | "lane", sourcePath: string, task?: string) =>
postJson<InspectUploadResponse>(`${API_BASE}/api/v1/data/inspect-upload`, {
project,
task: task || undefined,
source_path: sourcePath,
}),
};
export type BatchRecord = {
project: string;
task?: string;
batch: string;
pack?: string;
stage: string;
location: string;
path?: string;
counts?: { images?: number; labels?: number };
engineer?: string;
format?: string;
next_cli?: string;
};
export type PendingReport = {
workspace: string;
batches: BatchRecord[];
projects?: {
dms?: {
active_packs?: string[];
not_enabled?: string[];
task_defs?: Record<string, { type: string; nc?: number }>;
tasks?: Record<string, { inbox?: unknown[]; sources?: Record<string, unknown[]> }>;
recent_ingest?: { task: string; pack?: string; ts?: string; added?: number }[];
};
lane?: { packs?: Record<string, { path: string; train_lines?: number; enabled?: boolean }> };
};
};
export type CatalogReport = {
_cache?: {
cached?: boolean;
cache_source?: string;
cache_age_sec?: number;
build_source?: string;
};
dms?: Record<string, {
type: string;
nc?: number;
class_counts?: Record<string, number>;
packs?: {
name: string;
enabled: boolean;
train_images?: number;
val_images?: number;
test_images?: number;
class_counts?: Record<string, number>;
label_files?: number;
total_boxes?: number;
sampled?: boolean;
bbox_points?: [number, number][];
path?: string;
role?: string;
frozen?: boolean;
}[];
drop_paths?: { inbox?: string };
}>;
lane?: Record<string, {
path?: string;
drop_path?: string;
train_lines?: number;
val_lines?: number;
test_lines?: number;
enabled?: boolean;
add_template?: string;
quality?: {
analyzed_frames?: number;
lane_count_hist?: Record<string, number>;
length_hist?: { left: number; right: number; count: number }[];
curvature_hist?: { left: number; right: number; count: number }[];
};
}>;
};
export type ApprovalRecord = {
id: string;
action: string;
action_label?: string;
status: string;
params?: Record<string, unknown>;
note?: string | null;
submitted_by?: string;
submitted_at?: string;
review_comment?: string;
result?: { error?: string; ok?: boolean };
};
export type AuditImageItem = {
id: string;
batch: string;
location: string;
split: string;
filename: string;
has_label: boolean;
box_count: number;
missing_label: boolean;
};
export type AuditPreview = {
approval: ApprovalRecord;
scope_label?: string;
task?: string;
pack?: string;
class_names?: Record<number, string>;
batches?: { batch?: string; location?: string; path?: string; exists?: boolean }[];
};
export type JobRecord = {
id: string;
action: string;
status: string;
approval_id?: string;
params?: Record<string, unknown>;
created_at?: string;
started_at?: string;
finished_at?: string;
result?: { error?: string; ok?: boolean; [key: string]: unknown };
};
export type TrainingSummary = {
total: number;
running: number;
queued: number;
succeeded: number;
failed: number;
};
export type TrainingRecord = JobRecord & {
action_label?: string;
project?: string;
kind?: string;
task?: string | null;
track?: string;
weight_path?: string | null;
metrics?: Record<string, unknown>;
error?: string | null;
duration_sec?: number | null;
approval?: ApprovalRecord | null;
};
export type ModelRegistry = {
project: string;
task?: string;
version?: Record<string, unknown>;
tasks?: Record<string, Record<string, unknown>>;
eval_history?: Record<string, unknown>[];
};
export type DataCandidate = {
id: string;
project: string;
task?: string;
status: string;
source_type: string;
original_name?: string;
upload_path: string;
analyzed_source_path?: string;
format_id?: string;
split_counts?: Record<string, number>;
error_message?: string;
upload_size_bytes?: number;
submitted_by_name?: string;
analysis_job_id?: string;
created_at?: string;
updated_at?: string;
};
export type InspectUploadResponse = {
ok: boolean;
normalized: {
format_id: string;
project: string;
task?: string;
source_path: string;
split_counts?: Record<string, number>;
sample_count?: number;
annotation_count?: number;
artifacts?: string[];
warnings?: string[];
extra?: Record<string, unknown>;
};
};

View File

@@ -0,0 +1,120 @@
import { createContext, useCallback, useContext, useEffect, useMemo, useState, type ReactNode } from "react";
import { api, type AuthUser } from "../api/client";
const TOKEN_KEY = "as_access_token";
type AuthState = {
user: AuthUser | null;
loading: boolean;
token: string | null;
authConfig: { feishu_enabled: boolean; dev_auth_enabled: boolean } | null;
loginFeishu: () => void;
loginDev: (name?: string) => Promise<void>;
logout: () => void;
refreshUser: () => Promise<void>;
hasPermission: (code: string) => boolean;
};
const AuthCtx = createContext<AuthState | null>(null);
export function AuthProvider({ children }: { children: ReactNode }) {
const [user, setUser] = useState<AuthUser | null>(null);
const [loading, setLoading] = useState(true);
const [token, setToken] = useState<string | null>(() => localStorage.getItem(TOKEN_KEY));
const [authConfig, setAuthConfig] = useState<{ feishu_enabled: boolean; dev_auth_enabled: boolean } | null>(null);
useEffect(() => {
// 支持后端回调重定向到 /?token=...
const url = new URL(window.location.href);
const tokenFromQuery = url.searchParams.get("token");
if (tokenFromQuery) {
localStorage.setItem(TOKEN_KEY, tokenFromQuery);
api.setToken(tokenFromQuery);
url.searchParams.delete("token");
window.history.replaceState({}, "", url.toString());
setToken(tokenFromQuery);
}
}, []);
const applyToken = useCallback((t: string | null) => {
setToken(t);
if (t) localStorage.setItem(TOKEN_KEY, t);
else localStorage.removeItem(TOKEN_KEY);
api.setToken(t);
}, []);
const refreshUser = useCallback(async () => {
if (!token) {
setUser(null);
return;
}
try {
const me = await api.me();
setUser(me);
} catch {
applyToken(null);
setUser(null);
}
}, [token, applyToken]);
useEffect(() => {
api.authConfig().then(setAuthConfig).catch(() => setAuthConfig({ feishu_enabled: false, dev_auth_enabled: true }));
}, []);
useEffect(() => {
api.setToken(token);
if (token) {
refreshUser().finally(() => setLoading(false));
} else {
setUser(null);
setLoading(false);
}
}, [token, refreshUser]);
const loginFeishu = () => {
window.location.href = "/api/v1/auth/feishu/authorize";
};
const loginDev = async (name?: string) => {
const res = await api.devLogin(name);
applyToken(res.access_token);
setUser(res.user);
};
const logout = () => {
applyToken(null);
setUser(null);
};
const hasPermission = useCallback(
(code: string) => {
if (!user) return false;
const perms = user.permissions || [];
return perms.includes("*") || perms.includes(code);
},
[user]
);
const value = useMemo(
() => ({
user,
loading,
token,
loginFeishu,
loginDev,
logout,
refreshUser,
hasPermission,
authConfig,
}),
[user, loading, token, refreshUser, hasPermission, authConfig]
);
return <AuthCtx.Provider value={value}>{children}</AuthCtx.Provider>;
}
export function useAuth() {
const ctx = useContext(AuthCtx);
if (!ctx) throw new Error("useAuth outside AuthProvider");
return ctx;
}

View File

@@ -0,0 +1,18 @@
import { Navigate } from "react-router-dom";
import { useAuth } from "../auth/AuthContext";
import type { ReactNode } from "react";
export function RequireAuth({ children }: { children: ReactNode }) {
const { user, loading } = useAuth();
if (loading) return <p className="empty-state"></p>;
if (!user) return <Navigate to="/login" replace />;
return <>{children}</>;
}
export function RequirePermission({ code, children }: { code: string; children: ReactNode }) {
const { hasPermission } = useAuth();
if (!hasPermission(code)) {
return <p className="empty-state">访 {code}</p>;
}
return <>{children}</>;
}

View File

@@ -0,0 +1,111 @@
import { NavLink, Outlet } from "react-router-dom";
import { useEffect, useState } from "react";
import { api } from "../api/client";
import { useAuth } from "../auth/AuthContext";
const NAV = [
{ to: "/labeling", icon: "◎", label: "送标工作台", badge: "pending" as const, perm: "read:pending" },
{ to: "/catalog", icon: "▤", label: "数据目录", perm: "read:catalog" },
{ to: "/audit", icon: "✓", label: "审核管理", badge: "audit" as const, perm: "read:audit" },
{ to: "/jobs", icon: "◷", label: "任务监控", perm: "read:jobs" },
{ to: "/training", icon: "▶", label: "模型训练", perm: "write:approval_submit" },
{ to: "/logs", icon: "☰", label: "审计日志", perm: "read:audit" },
];
export function Layout() {
const { user, logout, hasPermission } = useAuth();
const [apiOk, setApiOk] = useState<boolean | null>(null);
const [pendingN, setPendingN] = useState(0);
const [auditN, setAuditN] = useState(0);
const refreshMeta = async () => {
try {
await api.health();
setApiOk(true);
if (hasPermission("read:pending")) {
const pending = await api.pending();
const actionable = (pending.batches || []).filter((b) =>
["returned", "raw_pool", "out_for_labeling"].includes(b.stage)
);
setPendingN(actionable.length);
}
if (hasPermission("read:audit")) {
const aud = await api.listApprovals("pending");
setAuditN(aud.items?.length || 0);
}
} catch {
setApiOk(false);
}
};
useEffect(() => {
refreshMeta();
const t = setInterval(refreshMeta, 30000);
return () => clearInterval(t);
}, [user]);
const path = location.pathname.replace(/^\//, "").split("/")[0] || "labeling";
const title = NAV.find((n) => n.to.slice(1) === path)?.label || "送标工作台";
const roleLabel = user?.roles?.map((r) => r.name).join(" · ") || "";
return (
<div className="app">
<aside className="sidebar">
<div className="brand">
<div className="brand-logo" aria-hidden="true">
<svg viewBox="0 0 40 40" fill="none">
<rect width="40" height="40" rx="10" fill="url(#g)" />
<path d="M12 28V12h6l4 10 4-10h6v16h-5V18l-4 10h-4l-4-10v10H12z" fill="white" fillOpacity="0.95" />
<defs>
<linearGradient id="g" x1="0" y1="0" x2="40" y2="40">
<stop stopColor="#0ea5e9" />
<stop offset="1" stopColor="#06b6d4" />
</linearGradient>
</defs>
</svg>
</div>
<div className="brand-text">
<span className="brand-company">Huaxu Sentinel</span>
<span className="brand-product">HSAP · </span>
</div>
</div>
<nav className="nav">
{NAV.filter((n) => hasPermission(n.perm)).map((n) => (
<NavLink key={n.to} to={n.to} className={({ isActive }) => "nav-item" + (isActive ? " active" : "")}>
<span className="nav-icon">{n.icon}</span> {n.label}
{n.badge === "pending" && pendingN > 0 && <span className="nav-badge">{pendingN}</span>}
{n.badge === "audit" && auditN > 0 && <span className="nav-badge">{auditN}</span>}
</NavLink>
))}
</nav>
<div className="sidebar-footer">
<div className="user-chip">
{user?.avatar_url ? <img src={user.avatar_url} alt="" className="user-avatar" /> : <span className="user-avatar user-avatar-ph">{user?.name?.[0]}</span>}
<div>
<div className="user-name">{user?.name}</div>
<div className="user-role text-dim">{roleLabel}</div>
</div>
</div>
<div className="env-chip">
<span className="env-dot" style={{ background: apiOk ? "var(--success)" : apiOk === false ? "var(--warning)" : undefined }} />
<span>{apiOk ? "算法服务运行中" : apiOk === false ? "算法服务离线" : "服务检测中…"}</span>
</div>
<button type="button" className="btn btn-sm btn-ghost btn-logout" onClick={logout}>退</button>
</div>
</aside>
<div className="main-wrap">
<header className="topbar">
<div className="topbar-title">
<h1>{title}</h1>
</div>
<div className="topbar-actions">
<button type="button" className="btn btn-ghost" onClick={() => refreshMeta()}></button>
</div>
</header>
<main className="content">
<Outlet context={{ refreshMeta }} />
</main>
</div>
</div>
);
}

View File

@@ -0,0 +1,30 @@
import { createContext, useCallback, useContext, useState, type ReactNode } from "react";
type Toast = { id: number; msg: string; err?: boolean };
const ToastCtx = createContext<(msg: string, err?: boolean) => void>(() => {});
export function ToastProvider({ children }: { children: ReactNode }) {
const [toasts, setToasts] = useState<Toast[]>([]);
const toast = useCallback((msg: string, err?: boolean) => {
const id = Date.now();
setToasts((t) => [...t, { id, msg, err }]);
setTimeout(() => setToasts((t) => t.filter((x) => x.id !== id)), 4000);
}, []);
return (
<ToastCtx.Provider value={toast}>
{children}
<div className="toast-container">
{toasts.map((t) => (
<div key={t.id} className={"toast" + (t.err ? " toast-err" : "")}>
{t.msg}
</div>
))}
</div>
</ToastCtx.Provider>
);
}
export function useToast() {
return useContext(ToastCtx);
}

View File

@@ -0,0 +1,2 @@
/** 车道线数据可视化(暂时关闭;恢复时改为 import.meta.env.VITE_LANE_DATA_VIZ === "1" */
export const LANE_DATA_VIZ_ENABLED = false;

View File

@@ -0,0 +1,21 @@
export const LabelingStates: Record<
string,
{ label: string; badge: string; hint: string }
> = {
raw_pool: { label: "原图池", badge: "badge-pending", hint: "仅有图像,待送标或等待标注回传" },
out_for_labeling: { label: "送标中", badge: "badge-training", hint: "已导出清单,等待标注方回传" },
returned: { label: "回传待入库", badge: "badge-staged", hint: "数据已落盘,可执行 build / add" },
ingested: { label: "已入库", badge: "badge-evaluated", hint: "已完成 ingest 或建包" },
};
export function forStage(stage: string) {
return LabelingStates[stage] ?? { label: stage, badge: "badge-idle", hint: "" };
}
export const DropPaths = {
dms_inbox: (ws: string, task: string, batch: string) =>
`${ws}/datasets/dms/inbox/${task}/${batch}/`,
dms_sources: (ws: string, pack: string, task: string, batch: string) =>
`${ws}/datasets/dms/packs/${pack}/${task}/sources/${batch}/`,
lane_add: (ws: string) => `${ws}/datasets/lane/ # archive 含 train_val_gt.txt`,
};

10
platform/web/src/main.tsx Normal file
View File

@@ -0,0 +1,10 @@
import { StrictMode } from "react";
import { createRoot } from "react-dom/client";
import App from "./App";
import "./styles/main.css";
createRoot(document.getElementById("root")!).render(
<StrictMode>
<App />
</StrictMode>
);

View File

@@ -0,0 +1,367 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { Link, useNavigate, useOutletContext, useParams } from "react-router-dom";
import { api, type ApprovalRecord, type AuditImageItem, type AuditPreview } from "../api/client";
import { useAuth } from "../auth/AuthContext";
import { useToast } from "../components/Toast";
const STATUS: Record<string, { label: string; badge: string }> = {
pending: { label: "待审核", badge: "badge-pending" },
approved: { label: "已通过", badge: "badge-staged" },
rejected: { label: "已驳回", badge: "badge-idle" },
executed: { label: "已执行", badge: "badge-evaluated" },
failed: { label: "执行失败", badge: "badge-pending" },
running: { label: "执行中", badge: "badge-training" },
};
type Ctx = { refreshMeta: () => void };
function AuthImage({
approvalId,
imageId,
thumb = true,
alt,
className,
onClick,
}: {
approvalId: string;
imageId: string;
thumb?: boolean;
alt: string;
className?: string;
onClick?: () => void;
}) {
const [src, setSrc] = useState<string | null>(null);
const [failed, setFailed] = useState(false);
useEffect(() => {
let revoked = false;
let objectUrl: string | null = null;
setFailed(false);
setSrc(null);
api.fetchApprovalImageBlob(approvalId, imageId, thumb).then(
(url) => {
if (revoked) {
URL.revokeObjectURL(url);
return;
}
objectUrl = url;
setSrc(url);
},
() => {
if (!revoked) setFailed(true);
}
);
return () => {
revoked = true;
if (objectUrl) URL.revokeObjectURL(objectUrl);
};
}, [approvalId, imageId, thumb]);
if (failed) {
return <div className={`audit-img-placeholder ${className || ""}`}></div>;
}
if (!src) {
return <div className={`audit-img-placeholder audit-img-loading ${className || ""}`}></div>;
}
return <img src={src} alt={alt} className={className} onClick={onClick} loading="lazy" />;
}
export function AuditDetailPage() {
const { id } = useParams<{ id: string }>();
const { refreshMeta } = useOutletContext<Ctx>();
const toast = useToast();
const { hasPermission } = useAuth();
const canReview = hasPermission("write:approval_review");
const [preview, setPreview] = useState<AuditPreview | null>(null);
const [images, setImages] = useState<AuditImageItem[]>([]);
const [total, setTotal] = useState(0);
const [loading, setLoading] = useState(true);
const [loadingMore, setLoadingMore] = useState(false);
const [lightbox, setLightbox] = useState<AuditImageItem | null>(null);
const [batchFilter, setBatchFilter] = useState("");
const sentinelRef = useRef<HTMLDivElement>(null);
const approvalId = id || "";
const loadPreview = useCallback(async () => {
if (!approvalId) return;
const data = await api.getApprovalPreview(approvalId);
setPreview(data);
}, [approvalId]);
const loadImages = useCallback(
async (append = false) => {
if (!approvalId) return;
if (append) setLoadingMore(true);
else setLoading(true);
try {
const offset = append ? images.length : 0;
const data = await api.listApprovalImages(approvalId, offset, 60);
setTotal(data.total);
setImages((prev) => (append ? [...prev, ...data.items] : data.items));
} finally {
setLoading(false);
setLoadingMore(false);
}
},
[approvalId, images.length]
);
useEffect(() => {
loadPreview().catch((e) => toast(String(e), true));
loadImages(false).catch((e) => toast(String(e), true));
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [approvalId]);
useEffect(() => {
const el = sentinelRef.current;
if (!el || loading || loadingMore || images.length >= total) return;
const obs = new IntersectionObserver(
(entries) => {
if (entries[0]?.isIntersecting && images.length < total) {
loadImages(true).catch((e) => toast(String(e), true));
}
},
{ rootMargin: "200px" }
);
obs.observe(el);
return () => obs.disconnect();
}, [images.length, total, loading, loadingMore, loadImages, toast]);
const rec: ApprovalRecord | undefined = preview?.approval;
const st = rec ? STATUS[rec.status] || { label: rec.status, badge: "badge-idle" } : null;
const batches = [...new Set(images.map((i) => i.batch))].sort();
const shown = batchFilter ? images.filter((i) => i.batch === batchFilter) : images;
return (
<>
<div className="audit-detail-head panel panel-compact">
<div className="panel-body">
<div className="audit-detail-nav">
<Link to="/audit" className="btn btn-sm btn-ghost"> </Link>
{rec && st && (
<span className={`badge ${st.badge}`}>{st.label}</span>
)}
</div>
{rec && (
<>
<h2 className="audit-detail-title">{rec.action_label || rec.action}</h2>
<p className="text-dim audit-detail-meta">
{rec.id} · {rec.submitted_by || "—"} · {rec.submitted_at?.slice(0, 19)}
</p>
{preview?.scope_label && (
<p className="audit-scope-label">{preview.scope_label}</p>
)}
{preview?.batches?.length ? (
<div className="audit-batch-tags">
{preview.batches.map((b) => (
<span key={`${b.location}:${b.batch}`} className={`audit-batch-tag ${b.exists ? "" : "missing"}`}>
{b.location}/{b.batch}
{!b.exists && " (目录不存在)"}
</span>
))}
</div>
) : null}
{rec.status === "pending" && canReview && (
<div className="audit-detail-actions">
<button
type="button"
className="btn btn-primary"
onClick={async () => {
try {
await api.approve(rec.id, "批准");
toast("已批准");
refreshMeta();
loadPreview();
} catch (e) {
toast(String(e), true);
}
}}
>
</button>
<button
type="button"
className="btn btn-ghost"
onClick={async () => {
const reason = prompt("驳回原因");
if (reason === null) return;
await api.reject(rec.id, reason);
toast("已驳回");
refreshMeta();
loadPreview();
}}
>
</button>
</div>
)}
</>
)}
</div>
</div>
<div className="panel">
<div className="panel-header">
<h2> · GT </h2>
<div className="audit-gallery-toolbar">
<span className="text-dim">
{images.length} / {total}
</span>
{batches.length > 1 && (
<select
className="audit-batch-select"
value={batchFilter}
onChange={(e) => setBatchFilter(e.target.value)}
>
<option value=""></option>
{batches.map((b) => (
<option key={b} value={b}>{b}</option>
))}
</select>
)}
</div>
</div>
<div className="panel-body">
{loading && !images.length ? (
<p className="empty-state"></p>
) : total === 0 ? (
<p className="empty-state"></p>
) : (
<>
<div className="audit-gallery">
{shown.map((item) => (
<button
key={item.id}
type="button"
className={`audit-gallery-item ${item.missing_label ? "no-label" : ""}`}
onClick={() => setLightbox(item)}
title={item.filename}
>
<AuthImage
approvalId={approvalId}
imageId={item.id}
thumb
alt={item.filename}
className="audit-gallery-img"
/>
<div className="audit-gallery-cap">
<span className="mono">{item.filename}</span>
<span>
{item.batch} · {item.split}
{item.box_count > 0 ? ` · ${item.box_count}` : item.missing_label ? " · 无标注" : ""}
</span>
</div>
</button>
))}
</div>
{loadingMore && <p className="empty-state"></p>}
<div ref={sentinelRef} className="audit-sentinel" />
</>
)}
</div>
</div>
{lightbox && (
<div className="audit-lightbox" onClick={() => setLightbox(null)} role="presentation">
<div className="audit-lightbox-inner" onClick={(e) => e.stopPropagation()}>
<button type="button" className="audit-lightbox-close btn btn-sm btn-ghost" onClick={() => setLightbox(null)}>
</button>
<AuthImage
approvalId={approvalId}
imageId={lightbox.id}
thumb={false}
alt={lightbox.filename}
className="audit-lightbox-img"
/>
<div className="audit-lightbox-meta">
<strong>{lightbox.filename}</strong>
<span>{lightbox.batch} · {lightbox.split} · {lightbox.box_count} </span>
</div>
</div>
</div>
)}
</>
);
}
export function AuditPage() {
const navigate = useNavigate();
const { refreshMeta } = useOutletContext<Ctx>();
const toast = useToast();
const { hasPermission } = useAuth();
const canReview = hasPermission("write:approval_review");
const [filter, setFilter] = useState("pending");
const [items, setItems] = useState<ApprovalRecord[]>([]);
const load = async () => {
const data = await api.listApprovals(filter || undefined);
setItems(data.items || []);
};
useEffect(() => { load(); }, [filter]);
return (
<div className="panel">
<div className="panel-header">
<h2></h2>
<div className="audit-filters">
{["pending", "executed", "rejected", "failed", ""].map((f) => (
<button key={f || "all"} type="button" className={`btn btn-sm ${filter === f ? "btn-primary" : "btn-ghost"}`} onClick={() => setFilter(f)}>
{f === "" ? "全部" : STATUS[f]?.label || f}
</button>
))}
</div>
</div>
<div className="panel-body table-wrap">
<table className="data-table audit-table">
<thead><tr><th></th><th></th><th></th><th></th><th></th><th></th><th></th></tr></thead>
<tbody>
{items.map((r) => {
const st = STATUS[r.status] || { label: r.status, badge: "badge-idle" };
return (
<tr
key={r.id}
className="audit-row-clickable"
onClick={() => navigate(`/audit/${r.id}`)}
>
<td className="mono text-sm">{r.id}</td>
<td>{r.action_label || r.action}</td>
<td><span className={`badge ${st.badge}`}>{st.label}</span></td>
<td>{r.submitted_by || "—"}</td>
<td className="text-sm">{r.submitted_at?.slice(0, 19)}</td>
<td className="text-sm"><pre className="params-pre">{JSON.stringify(r.params || {})}</pre></td>
<td onClick={(e) => e.stopPropagation()}>
<button type="button" className="btn btn-sm btn-ghost" onClick={() => navigate(`/audit/${r.id}`)}>
</button>
{r.status === "pending" && canReview ? (
<>
<button type="button" className="btn btn-sm btn-primary" onClick={async () => {
try { await api.approve(r.id, "批准"); toast("已批准"); load(); refreshMeta(); }
catch (e) { toast(String(e), true); }
}}></button>
<button type="button" className="btn btn-sm btn-ghost" onClick={async () => {
await api.reject(r.id, prompt("驳回原因") || "");
load(); refreshMeta();
}}></button>
</>
) : r.status === "pending" ? (
<span className="text-dim"></span>
) : (
r.review_comment || r.result?.error || ""
)}
</td>
</tr>
);
})}
</tbody>
</table>
</div>
</div>
);
}

View File

@@ -0,0 +1,22 @@
import { useEffect } from "react";
import { useNavigate, useSearchParams } from "react-router-dom";
import { api } from "../api/client";
export function AuthCallbackPage() {
const [params] = useSearchParams();
const navigate = useNavigate();
useEffect(() => {
const token = params.get("token");
if (token) {
localStorage.setItem("as_access_token", token);
api.setToken(token);
navigate("/labeling", { replace: true });
window.location.reload();
} else {
navigate("/login", { replace: true });
}
}, [params, navigate]);
return <p className="empty-state"></p>;
}

View File

@@ -0,0 +1,651 @@
import { useEffect, useMemo, useState } from "react";
import { api, type CatalogReport, type InspectUploadResponse } from "../api/client";
import { LANE_DATA_VIZ_ENABLED } from "../config/featureFlags";
type CatalogVersion = {
name: string;
enabled: boolean;
train_images?: number;
val_images?: number;
test_images?: number;
class_counts?: Record<string, number>;
path?: string;
role?: string;
frozen?: boolean;
label_files?: number;
total_boxes?: number;
sampled?: boolean;
bbox_points?: [number, number][];
lane_quality?: {
analyzed_frames?: number;
lane_count_hist?: Record<string, number>;
length_hist?: { left: number; right: number; count: number }[];
curvature_hist?: { left: number; right: number; count: number }[];
};
};
function polarToCartesian(cx: number, cy: number, r: number, angleDeg: number) {
const rad = (angleDeg - 90) * (Math.PI / 180);
return { x: cx + r * Math.cos(rad), y: cy + r * Math.sin(rad) };
}
function describeArc(cx: number, cy: number, r: number, startAngle: number, endAngle: number) {
const start = polarToCartesian(cx, cy, r, endAngle);
const end = polarToCartesian(cx, cy, r, startAngle);
const largeArcFlag = endAngle - startAngle <= 180 ? "0" : "1";
return `M ${start.x} ${start.y} A ${r} ${r} 0 ${largeArcFlag} 0 ${end.x} ${end.y}`;
}
function radarPoints(values: number[], cx = 110, cy = 110, radius = 76): string {
if (!values.length) return "";
return values
.map((v, i) => {
const angle = (-90 + (360 / values.length) * i) * (Math.PI / 180);
const r = Math.max(0, Math.min(1, v)) * radius;
const x = cx + r * Math.cos(angle);
const y = cy + r * Math.sin(angle);
return `${x},${y}`;
})
.join(" ");
}
export function CatalogPage() {
const [cat, setCat] = useState<CatalogReport | null>(null);
const [loading, setLoading] = useState(true);
const [loadError, setLoadError] = useState("");
const [cacheHint, setCacheHint] = useState("");
const [project, setProject] = useState<"dms" | "lane">("dms");
const [selectedTask, setSelectedTask] = useState<string>("");
const [selectedVersion, setSelectedVersion] = useState<string>("");
const [inspectProject, setInspectProject] = useState<"dms" | "lane">("dms");
const [inspectTask, setInspectTask] = useState<string>("dam");
const [inspectPath, setInspectPath] = useState<string>("");
const [inspecting, setInspecting] = useState(false);
const [inspectResult, setInspectResult] = useState<InspectUploadResponse["normalized"] | null>(null);
const [inspectError, setInspectError] = useState<string>("");
const load = async (refresh = false) => {
setLoading(true);
setLoadError("");
try {
const data = await api.catalog(refresh);
setCat(data);
const meta = data._cache;
if (meta?.cached) {
const src = meta.build_source === "reports" ? "报表缓存" : "扫描缓存";
setCacheHint(`${src} · ${Math.round(meta.cache_age_sec || 0)}s 前更新`);
} else if (refresh) {
setCacheHint("已强制刷新");
} else {
setCacheHint("已重新扫描");
}
} catch (e) {
setLoadError(e instanceof Error ? e.message : "加载失败");
} finally {
setLoading(false);
}
};
const inspectSource = async () => {
const sourcePath = inspectPath.trim();
if (!sourcePath) {
setInspectError("请先输入目录路径");
return;
}
setInspecting(true);
setInspectError("");
try {
const res = await api.inspectUploadPath(
inspectProject,
sourcePath,
inspectProject === "dms" ? inspectTask : undefined
);
setInspectResult(res.normalized);
} catch (e) {
setInspectResult(null);
setInspectError(e instanceof Error ? e.message : "目录分析失败");
} finally {
setInspecting(false);
}
};
useEffect(() => { load(); }, []);
useEffect(() => {
if (!cat) return;
const dmsTasks = Object.keys(cat.dms || {});
const lanePacks = Object.keys(cat.lane || {});
if (project === "dms") {
const first = selectedTask || dmsTasks[0] || "";
setSelectedTask(first);
const firstPack = (cat.dms?.[first]?.packs?.[0]?.name) || "";
setSelectedVersion(firstPack);
} else {
const first = selectedTask || lanePacks[0] || "";
setSelectedTask(first);
setSelectedVersion(first);
}
}, [cat, project]);
const dmsVersions = useMemo<CatalogVersion[]>(() => {
if (!cat || project !== "dms" || !selectedTask) return [];
return cat.dms?.[selectedTask]?.packs || [];
}, [cat, project, selectedTask]);
const laneVersions = useMemo<CatalogVersion[]>(() => {
if (!cat || project !== "lane") return [];
return Object.entries(cat.lane || {}).map(([name, info]) => ({
name,
enabled: Boolean(info.enabled),
train_images: info.train_lines || 0,
val_images: info.val_lines || 0,
test_images: info.test_lines || 0,
class_counts: {},
path: info.path,
lane_quality: info.quality,
}));
}, [cat, project]);
const versions: CatalogVersion[] = project === "dms" ? dmsVersions : laneVersions;
const current = versions.find((v) => v.name === selectedVersion) || versions[0];
const classPairs = Object.entries((current?.class_counts || {}) as Record<string, number>)
.sort((a, b) => b[1] - a[1])
.slice(0, 10);
const classMax = classPairs[0]?.[1] || 1;
const classTotal = classPairs.reduce((acc, [, v]) => acc + v, 0);
const versionFeatures = useMemo(() => {
if (project !== "dms") return [] as Array<{
name: string;
samples: number;
boxes: number;
density: number;
avg_area: number;
small_ratio: number;
bbox_points: [number, number][];
}>;
return dmsVersions.map((v) => {
const samples = (v.train_images || 0) + (v.val_images || 0) + (v.test_images || 0);
const boxes = v.total_boxes || 0;
const points = (v.bbox_points || []).filter((p) => p.length >= 2) as [number, number][];
const areas = points.map(([w, h]) => w * h);
const avg_area = areas.length ? areas.reduce((a, b) => a + b, 0) / areas.length : 0;
const small_ratio = areas.length ? areas.filter((a) => a < 0.02).length / areas.length : 0;
return {
name: v.name,
samples,
boxes,
density: samples > 0 ? boxes / samples : 0,
avg_area,
small_ratio,
bbox_points: points,
};
});
}, [project, dmsVersions]);
const featureMax = useMemo(() => {
const samples = Math.max(1, ...versionFeatures.map((v) => v.samples));
const density = Math.max(1, ...versionFeatures.map((v) => v.density));
const boxes = Math.max(1, ...versionFeatures.map((v) => v.boxes));
const avgArea = Math.max(1e-6, ...versionFeatures.map((v) => v.avg_area));
const smallRatio = Math.max(1e-6, ...versionFeatures.map((v) => v.small_ratio));
return { samples, density, boxes, avgArea, smallRatio };
}, [versionFeatures]);
const colorPalette = ["#22d3ee", "#a78bfa", "#34d399", "#f59e0b", "#60a5fa", "#f472b6", "#fb7185", "#94a3b8"];
const currentFeature = versionFeatures.find((v) => v.name === current?.name);
const whBins = useMemo(() => {
const bins = Array.from({ length: 10 }, () => Array.from({ length: 10 }, () => 0));
const points = currentFeature?.bbox_points || [];
for (const [w, h] of points) {
const xi = Math.min(9, Math.max(0, Math.floor(w * 10)));
const yi = Math.min(9, Math.max(0, Math.floor(h * 10)));
bins[9 - yi][xi] += 1;
}
let max = 0;
for (const row of bins) for (const v of row) max = Math.max(max, v);
return { bins, max };
}, [currentFeature]);
const radarAxes = ["样本量", "标签框", "框密度", "平均面积", "小目标占比"];
const radarMetrics = useMemo(() => {
return versionFeatures.slice(0, 4).map((v) => ({
name: v.name,
values: [
v.samples / featureMax.samples,
v.boxes / featureMax.boxes,
v.density / featureMax.density,
v.avg_area / featureMax.avgArea,
v.small_ratio / featureMax.smallRatio,
],
}));
}, [versionFeatures, featureMax]);
const laneCountPairs = useMemo(() => {
const hist = current?.lane_quality?.lane_count_hist || {};
const entries = Object.entries(hist);
entries.sort((a, b) => {
const av = a[0] === "8+" ? 999 : Number(a[0]);
const bv = b[0] === "8+" ? 999 : Number(b[0]);
return av - bv;
});
return entries;
}, [current]);
const laneLengthHist = current?.lane_quality?.length_hist || [];
const laneCurvHist = current?.lane_quality?.curvature_hist || [];
const laneLengthMax = Math.max(1, ...laneLengthHist.map((x) => x.count));
const laneCurvMax = Math.max(1, ...laneCurvHist.map((x) => x.count));
const kpis = useMemo(() => {
const allDms = Object.values(cat?.dms || {}) as NonNullable<CatalogReport["dms"]>[string][];
const allDmsPacks = allDms.flatMap((x) => x.packs || []);
const allLane = Object.values(cat?.lane || {}) as NonNullable<CatalogReport["lane"]>[string][];
const totalVersions = allDmsPacks.length + allLane.length;
const enabledVersions = allDmsPacks.filter((x) => x.enabled).length + allLane.filter((x) => x.enabled).length;
const totalSamples = allDmsPacks.reduce((acc, x) => acc + (x.train_images || 0) + (x.val_images || 0) + (x.test_images || 0), 0)
+ allLane.reduce((acc, x) => acc + (x.train_lines || 0) + (x.val_lines || 0) + (x.test_lines || 0), 0);
const totalBoxes = allDmsPacks.reduce((acc, x) => acc + (x.total_boxes || 0), 0);
return { totalVersions, enabledVersions, totalSamples, totalBoxes };
}, [cat]);
const inspectSplits = useMemo(() => {
const raw = inspectResult?.split_counts || {};
return [
["train", raw.train || 0],
["val", raw.val || 0],
["test", raw.test || 0],
] as Array<[string, number]>;
}, [inspectResult]);
const inspectTotal = inspectSplits.reduce((acc, [, v]) => acc + v, 0);
const fmt = (n: number) => n.toLocaleString("zh-CN");
if (loading && !cat) return <p className="empty-state"> catalog </p>;
if (loadError && !cat) return <p className="empty-state">{loadError}</p>;
if (!cat) return <p className="empty-state"></p>;
return (
<>
{cacheHint && <p className="audit-note" style={{ marginBottom: 8 }}>{cacheHint}</p>}
<div className="kpi-strip">
<div className="kpi"><span className="kpi-val">{fmt(kpis.totalVersions)}</span><span className="kpi-lbl"></span></div>
<div className="kpi"><span className="kpi-val">{fmt(kpis.enabledVersions)}</span><span className="kpi-lbl"></span></div>
<div className="kpi"><span className="kpi-val">{fmt(kpis.totalSamples)}</span><span className="kpi-lbl"></span></div>
<div className="kpi"><span className="kpi-val">{fmt(kpis.totalBoxes)}</span><span className="kpi-lbl">DMS </span></div>
</div>
<div className="panel">
<div className="panel-header"><h2></h2></div>
<div className="panel-body">
<div className="crud-form">
<label className="field">
<span>Project</span>
<select value={inspectProject} onChange={(e) => setInspectProject(e.target.value as "dms" | "lane")}>
<option value="dms">dms</option>
<option value="lane">lane</option>
</select>
</label>
{inspectProject === "dms" && (
<label className="field">
<span>Task</span>
<input value={inspectTask} onChange={(e) => setInspectTask(e.target.value)} placeholder="如 dam" />
</label>
)}
<label className="field" style={{ gridColumn: "1 / -1" }}>
<span></span>
<input
value={inspectPath}
onChange={(e) => setInspectPath(e.target.value)}
placeholder="例如 /data/workspace/DMS/inbox/dam/batch_xxx"
/>
</label>
<div className="crud-actions" style={{ gridColumn: "1 / -1" }}>
<button type="button" className="btn btn-primary" onClick={() => inspectSource()} disabled={inspecting}>
{inspecting ? "分析中..." : "分析目录"}
</button>
</div>
</div>
{inspectError && <p className="empty-state">{inspectError}</p>}
{inspectResult && (
<div className="feature-grid" style={{ marginTop: 12 }}>
<div className="feature-card">
<h4>Split </h4>
<div className="donut-wrap">
<svg viewBox="0 0 200 200" className="donut-svg" role="img" aria-label="目录 split 分布">
<circle cx="100" cy="100" r="70" fill="none" stroke="rgba(148,163,184,0.18)" strokeWidth="28" />
{(() => {
let angle = 0;
return inspectSplits.map(([name, value], idx) => {
const sweep = inspectTotal > 0 ? (value / inspectTotal) * 360 : 0;
const d = describeArc(100, 100, 70, angle, angle + sweep);
angle += sweep;
return <path key={name} d={d} fill="none" stroke={colorPalette[idx % colorPalette.length]} strokeWidth="28" />;
});
})()}
<text x="100" y="96" textAnchor="middle" className="donut-center-title">{inspectResult.format_id}</text>
<text x="100" y="116" textAnchor="middle" className="donut-center-sub">{fmt(inspectTotal)} samples</text>
</svg>
<div className="donut-legend">
{inspectSplits.map(([name, value], idx) => (
<div key={`inspect-${name}`} className="legend-item">
<i style={{ background: colorPalette[idx % colorPalette.length] }} />
<span>{name}</span>
<strong>{fmt(value)}</strong>
</div>
))}
</div>
</div>
</div>
<div className="feature-card">
<h4></h4>
<p className="audit-note">source: <code>{inspectResult.source_path}</code></p>
<p className="audit-note">annotations: {fmt(inspectResult.annotation_count || 0)}</p>
<p className="audit-note">artifacts: {(inspectResult.artifacts || []).join(", ") || "—"}</p>
{(inspectResult.warnings || []).length > 0 ? (
<ul className="audit-note">
{(inspectResult.warnings || []).map((w, idx) => <li key={`warn-${idx}`}>{w}</li>)}
</ul>
) : (
<p className="audit-note">warnings: none</p>
)}
</div>
</div>
)}
</div>
</div>
<div className="grid-2-equal">
<div className="panel">
<div className="panel-header">
<h2></h2>
<div className="audit-filters">
<button type="button" className={`btn btn-sm ${project === "dms" ? "btn-primary" : "btn-ghost"}`} onClick={() => setProject("dms")}>DMS</button>
<button type="button" className={`btn btn-sm ${project === "lane" ? "btn-primary" : "btn-ghost"}`} onClick={() => setProject("lane")}>Lane</button>
</div>
</div>
<div className="panel-body">
<div className="catalog-toolbar">
<label className="field">
<span>{project === "dms" ? "任务" : "包"}</span>
<select
value={selectedTask}
onChange={(e) => {
const value = e.target.value;
setSelectedTask(value);
const firstPack = project === "dms" ? (cat.dms?.[value]?.packs?.[0]?.name || "") : value;
setSelectedVersion(firstPack);
}}
>
{(project === "dms" ? Object.keys(cat.dms || {}) : Object.keys(cat.lane || {})).map((name) => (
<option key={name} value={name}>{name}</option>
))}
</select>
</label>
</div>
<div className="version-table">
<table className="data-table compact">
<thead><tr><th></th><th></th><th>Train</th><th>Val</th><th>Test</th><th></th></tr></thead>
<tbody>
{versions.map((v) => {
const total = (v.train_images || 0) + (v.val_images || 0) + (v.test_images || 0);
return (
<tr
key={v.name}
className={current?.name === v.name ? "row-active" : ""}
onClick={() => setSelectedVersion(v.name)}
style={{ cursor: "pointer" }}
>
<td><strong>{v.name}</strong></td>
<td><span className={`badge ${v.enabled ? "badge-promoted" : "badge-idle"}`}>{v.enabled ? "启用中" : "未启用"}</span></td>
<td>{fmt(v.train_images || 0)}</td>
<td>{fmt(v.val_images || 0)}</td>
<td>{fmt(v.test_images || 0)}</td>
<td>{fmt(total)}</td>
</tr>
);
})}
</tbody>
</table>
</div>
</div>
</div>
<div className="panel">
<div className="panel-header"><h2></h2></div>
<div className="panel-body">
{current ? (
<div>
<p className="catalog-intro">
<strong>{current.name}</strong> ·
Train {fmt(current.train_images || 0)} / Val {fmt(current.val_images || 0)} / Test {fmt(current.test_images || 0)}
</p>
{classPairs.length > 0 ? (
<div className="dist-list">
{classPairs.map(([name, value]) => (
<div key={name} className="dist-row">
<div className="dist-meta"><span>{name}</span><strong>{value}</strong></div>
<div className="progress-bar"><div className="progress-fill" style={{ width: `${Math.max(6, (value / classMax) * 100)}%` }} /></div>
</div>
))}
</div>
) : (
<p className="empty-state">Lane </p>
)}
</div>
) : <p className="empty-state"></p>}
</div>
</div>
</div>
<div className="panel">
<div className="panel-header"><h2></h2></div>
<div className="panel-body">
{project === "dms" ? (
<>
<div className="feature-grid">
<div className="feature-card">
<h4></h4>
{classPairs.length > 0 ? (
<div className="donut-wrap">
<svg viewBox="0 0 200 200" className="donut-svg" role="img" aria-label="类别结构环形图">
<circle cx="100" cy="100" r="70" fill="none" stroke="rgba(148,163,184,0.18)" strokeWidth="28" />
{(() => {
let angle = 0;
return classPairs.map(([name, value], idx) => {
const sweep = classTotal > 0 ? (value / classTotal) * 360 : 0;
const d = describeArc(100, 100, 70, angle, angle + sweep);
angle += sweep;
return <path key={name} d={d} fill="none" stroke={colorPalette[idx % colorPalette.length]} strokeWidth="28" strokeLinecap="butt" />;
});
})()}
<text x="100" y="96" textAnchor="middle" className="donut-center-title">{current?.name || "-"}</text>
<text x="100" y="116" textAnchor="middle" className="donut-center-sub">{fmt(classTotal)} boxes</text>
</svg>
<div className="donut-legend">
{classPairs.slice(0, 6).map(([name, value], idx) => (
<div key={`legend-${name}`} className="legend-item">
<i style={{ background: colorPalette[idx % colorPalette.length] }} />
<span>{name}</span>
<strong>{((value / Math.max(1, classTotal)) * 100).toFixed(1)}%</strong>
</div>
))}
</div>
</div>
) : <p className="empty-state"></p>}
</div>
<div className="feature-card">
<h4></h4>
<svg viewBox="0 0 320 220" className="scatter-svg" role="img" aria-label="目标宽高散点密度分布">
<rect x="40" y="20" width="250" height="170" fill="rgba(148,163,184,0.05)" stroke="rgba(148,163,184,0.25)" />
{whBins.bins.map((row, yi) =>
row.map((v, xi) => {
if (!v) return null;
const alpha = Math.min(0.85, v / Math.max(1, whBins.max));
return (
<rect
key={`bin-${xi}-${yi}`}
x={40 + xi * 25}
y={20 + yi * 17}
width={25}
height={17}
fill={`rgba(34,211,238,${alpha.toFixed(3)})`}
/>
);
})
)}
{(currentFeature?.bbox_points || []).slice(0, 400).map(([w, h], idx) => (
<circle
key={`wh-${idx}`}
cx={40 + w * 250}
cy={190 - h * 170}
r={1.6}
fill="rgba(255,255,255,0.35)"
/>
))}
<text x="294" y="205" textAnchor="end" className="scatter-axis"> w</text>
<text x="12" y="20" className="scatter-axis"> h</text>
</svg>
</div>
<div className="feature-card">
<h4></h4>
<div className="radar-wrap">
<svg viewBox="0 0 220 220" className="radar-svg" role="img" aria-label="版本多指标雷达图">
{[0.25, 0.5, 0.75, 1].map((lv) => (
<polygon
key={`grid-${lv}`}
points={radarPoints([lv, lv, lv, lv, lv], 110, 110, 76)}
fill="none"
stroke="rgba(148,163,184,0.2)"
strokeWidth="1"
/>
))}
{radarAxes.map((label, i) => {
const pt = radarPoints([1, 1, 1, 1, 1], 110, 110, 90).split(" ")[i];
const [x, y] = pt.split(",");
return <text key={label} x={Number(x)} y={Number(y)} className="radar-axis">{label}</text>;
})}
{radarMetrics.map((m, idx) => (
<polygon
key={`radar-${m.name}`}
points={radarPoints(m.values, 110, 110, 76)}
fill={colorPalette[idx % colorPalette.length]}
fillOpacity="0.16"
stroke={colorPalette[idx % colorPalette.length]}
strokeWidth="2"
/>
))}
</svg>
<div className="donut-legend">
{radarMetrics.map((m, idx) => (
<div key={`radar-leg-${m.name}`} className="legend-item">
<i style={{ background: colorPalette[idx % colorPalette.length] }} />
<span>{m.name}</span>
</div>
))}
</div>
</div>
</div>
</div>
<div className="version-table" style={{ marginTop: 14 }}>
<table className="data-table compact">
<thead><tr><th></th><th>Train</th><th>Val</th><th>Test</th><th></th><th>/</th><th></th></tr></thead>
<tbody>
{versionFeatures.map((v) => {
const raw = dmsVersions.find((x) => x.name === v.name);
return (
<tr key={v.name}>
<td>{v.name}</td>
<td>{fmt(raw?.train_images || 0)}</td>
<td>{fmt(raw?.val_images || 0)}</td>
<td>{fmt(raw?.test_images || 0)}</td>
<td>{fmt(v.boxes)}</td>
<td>{v.density.toFixed(2)}</td>
<td>{(v.avg_area * 100).toFixed(2)}%</td>
</tr>
);
})}
</tbody>
</table>
</div>
</>
) : LANE_DATA_VIZ_ENABLED ? (
<>
<div className="feature-grid">
<div className="feature-card">
<h4>线</h4>
<div className="donut-wrap">
<svg viewBox="0 0 200 200" className="donut-svg" role="img" aria-label="每帧车道线数量分布">
<circle cx="100" cy="100" r="70" fill="none" stroke="rgba(148,163,184,0.18)" strokeWidth="28" />
{(() => {
const total = laneCountPairs.reduce((a, [, c]) => a + c, 0);
let angle = 0;
return laneCountPairs.map(([bucket, cnt], idx) => {
const sweep = total > 0 ? (cnt / total) * 360 : 0;
const d = describeArc(100, 100, 70, angle, angle + sweep);
angle += sweep;
return <path key={bucket} d={d} fill="none" stroke={colorPalette[idx % colorPalette.length]} strokeWidth="28" />;
});
})()}
<text x="100" y="100" textAnchor="middle" className="donut-center-title">线</text>
</svg>
<div className="donut-legend">
{laneCountPairs.map(([bucket, cnt], idx) => (
<div key={`lc-${bucket}`} className="legend-item">
<i style={{ background: colorPalette[idx % colorPalette.length] }} />
<span>{bucket} </span>
<strong>{fmt(cnt)}</strong>
</div>
))}
</div>
</div>
</div>
<div className="feature-card">
<h4>线线</h4>
<svg viewBox="0 0 320 200" className="scatter-svg" role="img" aria-label="线长度分布曲线">
<line x1="36" y1="170" x2="300" y2="170" stroke="rgba(148,163,184,0.35)" />
<line x1="36" y1="20" x2="36" y2="170" stroke="rgba(148,163,184,0.35)" />
{laneLengthHist.length > 0 && (
<polyline
fill="none"
stroke="#22d3ee"
strokeWidth="2"
points={laneLengthHist.map((b, i) => {
const x = 36 + (i / Math.max(1, laneLengthHist.length - 1)) * 264;
const y = 170 - (b.count / laneLengthMax) * 140;
return `${x},${y}`;
}).join(" ")}
/>
)}
<text x="300" y="188" textAnchor="end" className="scatter-axis"></text>
<text x="8" y="20" className="scatter-axis"></text>
</svg>
</div>
<div className="feature-card">
<h4>线</h4>
<svg viewBox="0 0 320 200" className="scatter-svg" role="img" aria-label="曲率分布曲线">
<line x1="36" y1="170" x2="300" y2="170" stroke="rgba(148,163,184,0.35)" />
<line x1="36" y1="20" x2="36" y2="170" stroke="rgba(148,163,184,0.35)" />
{laneCurvHist.length > 0 && (
<polyline
fill="none"
stroke="#a78bfa"
strokeWidth="2"
points={laneCurvHist.map((b, i) => {
const x = 36 + (i / Math.max(1, laneCurvHist.length - 1)) * 264;
const y = 170 - (b.count / laneCurvMax) * 140;
return `${x},${y}`;
}).join(" ")}
/>
)}
<text x="300" y="188" textAnchor="end" className="scatter-axis"></text>
<text x="8" y="20" className="scatter-axis"></text>
</svg>
</div>
</div>
</>
) : (
<p className="empty-state">线/</p>
)}
<div className="crud-actions" style={{ marginTop: 14 }}>
<button type="button" className="btn btn-ghost" onClick={() => load(true)} disabled={loading}>
{loading ? "刷新中..." : "刷新数据"}
</button>
<button type="button" className="btn btn-ghost" onClick={() => setSelectedVersion(versions[0]?.name || "")}></button>
</div>
</div>
</div>
</>
);
}

View File

@@ -0,0 +1,262 @@
import { useEffect, useMemo, useState } from "react";
import { useOutletContext } from "react-router-dom";
import { api, type CatalogReport, type JobRecord, type PendingReport } from "../api/client";
import { useToast } from "../components/Toast";
import { LANE_DATA_VIZ_ENABLED } from "../config/featureFlags";
type Ctx = { refreshMeta: () => void };
type IterAction = "train_dms" | "train_lane" | "eval_dms" | "eval_lane" | "visualize_dms" | "visualize_lane";
const ITERATE_ACTIONS = new Set<IterAction>([
"train_dms",
"train_lane",
"eval_dms",
"eval_lane",
"visualize_dms",
"visualize_lane",
]);
const ACTION_META: Record<IterAction, { label: string; project: "DMS" | "Lane"; kind: "训练" | "评估" | "可视化" }> = {
train_dms: { label: "DMS 训练", project: "DMS", kind: "训练" },
train_lane: { label: "Lane 训练", project: "Lane", kind: "训练" },
eval_dms: { label: "DMS 评估", project: "DMS", kind: "评估" },
eval_lane: { label: "Lane 评估", project: "Lane", kind: "评估" },
visualize_dms: { label: "DMS 可视化", project: "DMS", kind: "可视化" },
visualize_lane: { label: "Lane 可视化", project: "Lane", kind: "可视化" },
};
function statusBadgeClass(status: string): string {
if (status === "succeeded") return "badge-promoted";
if (status === "running") return "badge-training";
if (status === "failed") return "badge-pending";
return "badge-idle";
}
function fmtTime(ts?: string): string {
if (!ts) return "—";
const d = new Date(ts);
if (Number.isNaN(d.getTime())) return ts;
return d.toLocaleString("zh-CN", { hour12: false });
}
function extractWeight(job: JobRecord): string | null {
const res = (job.result || {}) as Record<string, unknown>;
const params = (job.params || {}) as Record<string, unknown>;
const candidates = [res.best_weights, res.candidate, res.model_path, res.weights, params.weights, params.model_path];
for (const v of candidates) {
if (typeof v === "string" && v.trim()) return v;
}
return null;
}
export function IteratePage() {
const { refreshMeta } = useOutletContext<Ctx>();
const toast = useToast();
const [pending, setPending] = useState<PendingReport | null>(null);
const [cat, setCat] = useState<CatalogReport | null>(null);
const [jobs, setJobs] = useState<JobRecord[]>([]);
const [dmsTask, setDmsTask] = useState("dam");
const [dmsTrack, setDmsTrack] = useState<"platform" | "local">("platform");
const [dmsWeights, setDmsWeights] = useState("");
const [laneTrack, setLaneTrack] = useState<"platform" | "local">("platform");
const [laneModelPath, setLaneModelPath] = useState("");
const [laneDataRoot, setLaneDataRoot] = useState("");
const [laneTestList, setLaneTestList] = useState("list/test_gt.txt");
const [selectedJobId, setSelectedJobId] = useState<string | null>(null);
const loadAll = async () => {
const [p, c, j] = await Promise.all([api.pending(), api.catalog(), api.listJobs()]);
setPending(p);
setCat(c);
setJobs(j.items || []);
};
useEffect(() => { loadAll(); }, []);
useEffect(() => {
const tasks = Object.keys(pending?.projects?.dms?.task_defs || {});
if (tasks.length > 0 && !tasks.includes(dmsTask)) {
setDmsTask(tasks[0]);
}
}, [pending]);
const submit = async (action: string, params: Record<string, unknown>) => {
try {
await api.submitApproval(action, params);
toast(`已提交 ${action}`);
refreshMeta();
await loadAll();
} catch (e) {
toast(String(e), true);
}
};
const packRows = [
...(pending?.projects?.dms?.active_packs || []).map((n) => ({ project: "dms", name: n, enabled: true })),
...(pending?.projects?.dms?.not_enabled || []).map((n) => ({ project: "dms", name: n, enabled: false })),
...Object.entries(cat?.lane || {}).map(([n, info]) => ({ project: "lane", name: n, enabled: info.enabled })),
];
const iterJobs = useMemo(() => {
return jobs.filter((j) => ITERATE_ACTIONS.has(j.action as IterAction)).slice(0, 30);
}, [jobs]);
const milestones = useMemo(() => [...iterJobs].reverse(), [iterJobs]);
const selectedJob = useMemo(
() => milestones.find((j) => j.id === selectedJobId) || milestones[milestones.length - 1] || null,
[milestones, selectedJobId]
);
useEffect(() => {
if (!selectedJobId && milestones.length > 0) {
setSelectedJobId(milestones[milestones.length - 1].id);
}
}, [milestones, selectedJobId]);
const dmsTasks = Object.keys(pending?.projects?.dms?.task_defs || {});
return (
<>
<div className="grid-2">
<div className="panel">
<div className="panel-header"><h2>DMSYOLO//</h2></div>
<div className="panel-body">
<div className="crud-form">
<label className="field">
<span></span>
<select value={dmsTask} onChange={(e) => setDmsTask(e.target.value)}>
{(dmsTasks.length > 0 ? dmsTasks : [dmsTask]).map((t) => <option key={t} value={t}>{t}</option>)}
</select>
</label>
<label className="field">
<span></span>
<select value={dmsTrack} onChange={(e) => setDmsTrack(e.target.value as "platform" | "local")}>
<option value="platform">platform</option>
<option value="local">local</option>
</select>
</label>
<label className="field" style={{ gridColumn: "1 / -1" }}>
<span>/</span>
<input value={dmsWeights} onChange={(e) => setDmsWeights(e.target.value)} placeholder="/path/to/best.pt" />
</label>
</div>
<div className="audit-quick">
<button type="button" className="btn btn-sm btn-primary" onClick={() => submit("train_dms", { task: dmsTask, mode: "full", track: dmsTrack })}></button>
<button type="button" className="btn btn-sm btn-ghost" onClick={() => submit("eval_dms", { task: dmsTask, ...(dmsWeights ? { weights: dmsWeights } : {}) })}></button>
<button type="button" className="btn btn-sm btn-ghost" onClick={() => submit("visualize_dms", { task: dmsTask, ...(dmsWeights ? { weights: dmsWeights } : {}) })}></button>
</div>
</div>
</div>
<div className="panel">
<div className="panel-header"><h2>LaneUFLD/{LANE_DATA_VIZ_ENABLED ? "/可视化" : ""}</h2></div>
<div className="panel-body">
<div className="crud-form">
<label className="field">
<span></span>
<select value={laneTrack} onChange={(e) => setLaneTrack(e.target.value as "platform" | "local")}>
<option value="platform">platform</option>
<option value="local">local</option>
</select>
</label>
<label className="field">
<span>test_list</span>
<input value={laneTestList} onChange={(e) => setLaneTestList(e.target.value)} placeholder="list/test_gt.txt" />
</label>
<label className="field" style={{ gridColumn: "1 / -1" }}>
<span>/</span>
<input value={laneModelPath} onChange={(e) => setLaneModelPath(e.target.value)} placeholder="/path/to/best.pth" />
</label>
<label className="field" style={{ gridColumn: "1 / -1" }}>
<span>data_root</span>
<input value={laneDataRoot} onChange={(e) => setLaneDataRoot(e.target.value)} placeholder="/path/to/lane/dataset/root" />
</label>
</div>
<div className="audit-quick">
<button type="button" className="btn btn-sm btn-primary" onClick={() => submit("train_lane", { track: laneTrack })}></button>
<button type="button" className="btn btn-sm btn-ghost" onClick={() => submit("eval_lane", { model_path: laneModelPath, ...(laneDataRoot ? { data_root: laneDataRoot } : {}), test_list: laneTestList })}></button>
{LANE_DATA_VIZ_ENABLED && (
<button type="button" className="btn btn-sm btn-ghost" onClick={() => submit("visualize_lane", { model_path: laneModelPath, ...(laneDataRoot ? { data_root: laneDataRoot } : {}), test_list: laneTestList })}></button>
)}
</div>
</div>
</div>
</div>
<div className="panel">
<div className="panel-header"><h2></h2></div>
<div className="panel-body pack-list">
{packRows.map((p) => (
<div key={`${p.project}/${p.name}`} className={`pack-row ${p.enabled ? "active-pack" : ""}`}>
<span className="pack-name">{p.project}/{p.name}</span>
<span className={`badge ${p.enabled ? "badge-staged" : "badge-idle"}`}>{p.enabled ? "已启用" : "未启用"}</span>
<code className="text-sm">python as.py enable {p.project} {p.name}</code>
</div>
))}
</div>
</div>
<div className="panel">
<div className="panel-header"><h2></h2></div>
<div className="panel-body">
{milestones.length === 0 ? (
<div className="empty-state">/</div>
) : (
<div className="milestone-layout">
<div className="milestone-track">
{milestones.map((j) => {
const action = j.action as IterAction;
const meta = ACTION_META[action];
const weight = extractWeight(j);
return (
<button
type="button"
key={j.id}
className={`milestone-node ${selectedJob?.id === j.id ? "active" : ""}`}
onClick={() => setSelectedJobId(j.id)}
>
<div className="milestone-dot" />
<div className="milestone-body">
<div className="milestone-title">{meta?.label || j.action}</div>
<div className="milestone-sub">{fmtTime(j.started_at || j.created_at)}</div>
<div className="milestone-sub">
<span className={`badge ${statusBadgeClass(j.status)}`}>{j.status}</span>
<span>{meta?.project || "未知"}</span>
<span>{meta?.kind || "动作"}</span>
</div>
<div className="milestone-sub mono">{weight ? `权重: ${weight}` : "权重: —"}</div>
</div>
</button>
);
})}
</div>
{selectedJob && (
<div className="milestone-detail">
<div className="milestone-detail-head">
<h3>{ACTION_META[selectedJob.action as IterAction]?.label || selectedJob.action}</h3>
<span className={`badge ${statusBadgeClass(selectedJob.status)}`}>{selectedJob.status}</span>
</div>
<dl className="detail-dl">
<dt>Job ID</dt>
<dd className="mono">{selectedJob.id}</dd>
<dt></dt>
<dd>{fmtTime(selectedJob.created_at)}</dd>
<dt></dt>
<dd>{fmtTime(selectedJob.started_at)}</dd>
<dt></dt>
<dd>{fmtTime(selectedJob.finished_at)}</dd>
<dt></dt>
<dd className="path-dd">{extractWeight(selectedJob) || "—"}</dd>
<dt></dt>
<dd><pre className="params-pre">{JSON.stringify(selectedJob.params || {}, null, 2)}</pre></dd>
<dt></dt>
<dd><pre className="params-pre milestone-result">{JSON.stringify(selectedJob.result || {}, null, 2)}</pre></dd>
</dl>
</div>
)}
</div>
)}
</div>
</div>
</>
);
}

View File

@@ -0,0 +1,33 @@
import { useEffect, useState } from "react";
import { api, type JobRecord } from "../api/client";
export function JobsPage() {
const [items, setItems] = useState<JobRecord[]>([]);
useEffect(() => { api.listJobs().then((d) => setItems(d.items || [])); }, []);
const badge = (s: string) =>
({ queued: "badge-pending", running: "badge-training", succeeded: "badge-evaluated", failed: "badge-pending" }[s] || "badge-idle");
return (
<div className="panel">
<div className="panel-header"><h2>Job </h2><span className="text-dim">{items.length} </span></div>
<div className="panel-body table-wrap">
<table className="data-table">
<thead><tr><th>Job ID</th><th></th><th></th><th></th><th></th><th></th></tr></thead>
<tbody>
{items.map((j) => (
<tr key={j.id}>
<td className="mono text-sm">{j.id}</td>
<td>{j.action}</td>
<td><span className={`badge ${badge(j.status)}`}>{j.status}</span></td>
<td>{j.approval_id || "—"}</td>
<td className="text-sm">{j.started_at?.slice(0, 19)}</td>
<td className="text-sm">{j.result?.error || (j.result?.ok ? "ok" : "")}</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
);
}

View File

@@ -0,0 +1,161 @@
import { useEffect, useMemo, useState } from "react";
import { useOutletContext } from "react-router-dom";
import { api, type BatchRecord, type PendingReport } from "../api/client";
import { forStage } from "../lib/labeling";
import { useToast } from "../components/Toast";
import { UploadsPage } from "./Uploads";
type Ctx = { refreshMeta: () => void };
export function LabelingPage() {
const { refreshMeta } = useOutletContext<Ctx>();
const toast = useToast();
const [pending, setPending] = useState<PendingReport | null>(null);
const [project, setProject] = useState<"dms" | "lane">("dms");
const [selectedTask, setSelectedTask] = useState<string | null>(null);
const [selectedPack, setSelectedPack] = useState<string | null>(null);
const [selectedBatchId, setSelectedBatchId] = useState<string | null>(null);
const load = async () => {
setPending(await api.pending());
};
useEffect(() => {
load();
}, []);
const batches = useMemo(() => {
const all = pending?.batches || [];
if (project === "dms" && selectedTask) return all.filter((b) => b.project === "dms" && b.task === selectedTask);
if (project === "lane" && selectedPack) {
return all.filter((b) => b.project === "lane" && (b.pack === selectedPack || b.batch === selectedPack));
}
return all;
}, [pending, project, selectedTask, selectedPack]);
const selected = batches.find((b) => `${b.location}:${b.batch}` === selectedBatchId) || batches[0];
if (!pending) return <p className="empty-state"></p>;
const bs = pending?.batches || [];
const dmsTasks = Object.keys(pending.projects?.dms?.task_defs || {});
const lanePacks = Object.keys(pending.projects?.lane?.packs || {});
return (
<>
<div className="kpi-strip">
<div className="kpi"><span className="kpi-val">{bs.filter((b) => b.stage === "returned").length}</span><span className="kpi-lbl"></span></div>
<div className="kpi"><span className="kpi-val">{bs.filter((b) => b.stage === "raw_pool").length}</span><span className="kpi-lbl"></span></div>
<div className="kpi"><span className="kpi-val">{bs.filter((b) => b.stage === "out_for_labeling").length}</span><span className="kpi-lbl"></span></div>
<div className="kpi"><span className="kpi-val">{(pending?.projects?.dms?.active_packs || []).length}</span><span className="kpi-lbl"></span></div>
</div>
<div className="panel">
<div className="panel-header"><h2></h2></div>
<div className="panel-body">
<UploadsPage embedded />
</div>
</div>
<div className="panel">
<div className="panel-header">
<h2></h2>
<div className="audit-filters">
<button type="button" className={`btn btn-sm ${project === "dms" ? "btn-primary" : "btn-ghost"}`} onClick={() => { setProject("dms"); setSelectedPack(null); }}>DMS</button>
<button type="button" className={`btn btn-sm ${project === "lane" ? "btn-primary" : "btn-ghost"}`} onClick={() => { setProject("lane"); setSelectedTask(null); }}>Lane</button>
</div>
</div>
<div className="panel-body">
<div className="catalog-toolbar">
<label className="field">
<span>{project === "dms" ? "任务" : "数据包"}</span>
<select
value={project === "dms" ? (selectedTask || "") : (selectedPack || "")}
onChange={(e) => {
const value = e.target.value;
if (project === "dms") setSelectedTask(value || null);
else setSelectedPack(value || null);
setSelectedBatchId(null);
}}
>
<option value=""></option>
{(project === "dms" ? dmsTasks : lanePacks).map((name) => (
<option key={name} value={name}>{name}</option>
))}
</select>
</label>
</div>
<div className="grid-2-equal">
<div className="panel panel-compact">
<div className="panel-header"><h2></h2><span className="text-dim">{batches.length} </span></div>
<div className="panel-body table-wrap">
<table className="data-table">
<thead><tr><th></th><th></th><th></th><th></th><th></th></tr></thead>
<tbody>
{batches.map((b) => {
const st = forStage(b.stage);
const id = `${b.location}:${b.batch}`;
return (
<tr key={id} className={selected === b ? "row-active" : ""} onClick={() => setSelectedBatchId(id)} style={{ cursor: "pointer" }}>
<td><strong>{b.batch}</strong></td>
<td><span className={`badge ${st.badge}`}>{st.label}</span></td>
<td>{b.location}</td>
<td>{b.counts?.images ?? "—"}</td>
<td>{b.counts?.labels ?? "—"}</td>
</tr>
);
})}
</tbody>
</table>
</div>
</div>
<div className="panel panel-compact">
<div className="panel-header"><h2></h2></div>
<div className="panel-body">
{selected
? <BatchDetail b={selected} onDone={() => { load(); refreshMeta(); toast("已提交"); }} />
: <p className="empty-state"></p>}
</div>
</div>
</div>
</div>
</div>
</>
);
}
function BatchDetail({ b, onDone }: { b: BatchRecord; onDone: () => void }) {
const st = forStage(b.stage);
return (
<div className="batch-detail">
<div className="batch-detail-header">
<h3>{b.batch}</h3>
<span className={`badge ${st.badge}`}>{st.label}</span>
</div>
<dl className="detail-dl">
<dt></dt><dd className="mono path-dd">{b.path || "由上传解析后生成"}</dd>
<dt></dt><dd>{b.location}{b.pack ? ` · ${b.pack}` : ""}</dd>
<dt> / </dt><dd>{b.counts?.images ?? "—"} / {b.counts?.labels ?? "—"}</dd>
</dl>
{b.next_cli && (
<div className="cli-box">
<code>{b.next_cli}</code>
<button type="button" className="btn btn-sm btn-ghost" onClick={() => navigator.clipboard.writeText(b.next_cli!)}></button>
</div>
)}
<div className="batch-actions">
{b.project === "dms" && b.stage === "returned" && (
<button type="button" className="btn btn-sm btn-primary" onClick={async () => {
await api.submitBuildBatch({ task: b.task, batch: b.batch, pack: b.pack || "dms_v2", location: b.location });
onDone();
}}></button>
)}
<button type="button" className="btn btn-sm btn-ghost" onClick={async () => {
await api.registerBatch({ project: b.project, task: b.task, batch: b.batch, pack: b.pack, location: b.location });
onDone();
}}> meta</button>
</div>
</div>
);
}

View File

@@ -0,0 +1,52 @@
import { useEffect, useState } from "react";
import { useNavigate } from "react-router-dom";
import { useAuth } from "../auth/AuthContext";
export function LoginPage() {
const { user, loginFeishu, loginDev, authConfig } = useAuth();
const navigate = useNavigate();
const [err, setErr] = useState("");
const [name, setName] = useState("开发用户");
useEffect(() => {
if (user) navigate("/labeling", { replace: true });
}, [user, navigate]);
const onDevLogin = async () => {
setErr("");
try {
await loginDev(name);
navigate("/labeling");
} catch (e) {
setErr(String(e));
}
};
return (
<div className="login-page">
<div className="login-shell simple">
<section className="login-card panel" id="access">
<h2></h2>
<p className="text-dim">广 · AEB </p>
{authConfig?.feishu_enabled && (
<button type="button" className="btn btn-primary btn-feishu" onClick={loginFeishu}>
使
</button>
)}
{authConfig?.dev_auth_enabled && (
<div className="dev-login">
<p className="text-sm text-dim"> App</p>
<input type="text" value={name} onChange={(e) => setName(e.target.value)} placeholder="显示名称" />
<button type="button" className="btn btn-ghost" onClick={onDevLogin}></button>
</div>
)}
{!authConfig?.feishu_enabled && !authConfig?.dev_auth_enabled && (
<p className="empty-state"></p>
)}
{err && <p className="login-err">{err}</p>}
</section>
</div>
</div>
);
}

View File

@@ -0,0 +1,30 @@
import { useEffect, useState } from "react";
import { api, type PendingReport } from "../api/client";
export function LogsPage() {
const [pending, setPending] = useState<PendingReport | null>(null);
useEffect(() => { api.pending().then(setPending); }, []);
const recent = pending?.projects?.dms?.recent_ingest || [];
return (
<>
<div className="panel">
<div className="panel-header"><h2>ingest_log.jsonl</h2></div>
<div className="panel-body log-list">
{recent.length ? recent.map((l, i) => (
<div key={i} className="log-entry">
<span className="log-time">{l.ts}</span>
<span className="log-type ingest">ingest</span>
<span>task={l.task} pack={l.pack} added={l.added ?? "—"}</span>
</div>
)) : <p className="empty-state"></p>}
</div>
</div>
<div className="panel">
<div className="panel-header"><h2>approval_queue.jsonl</h2></div>
<div className="panel-body"><p className="mono text-sm">HSAP/manifests/approval_queue.jsonl</p></div>
</div>
</>
);
}

View File

@@ -0,0 +1,588 @@
import { useCallback, useEffect, useMemo, useState } from "react";
import { Link, useOutletContext } from "react-router-dom";
import {
api,
type ModelRegistry,
type PendingReport,
type TrainingRecord,
} from "../api/client";
import { useToast } from "../components/Toast";
import { LANE_DATA_VIZ_ENABLED } from "../config/featureFlags";
type Ctx = { refreshMeta: () => void };
type TrainAction =
| "train_dms"
| "train_lane"
| "eval_dms"
| "eval_lane"
| "visualize_dms"
| "visualize_lane"
| "promote_dms";
const CREATE_ACTIONS: { id: TrainAction; label: string; project: "dms" | "lane"; kind: string }[] = [
{ id: "train_dms", label: "DMS 训练", project: "dms", kind: "train" },
{ id: "eval_dms", label: "DMS 评估", project: "dms", kind: "eval" },
{ id: "visualize_dms", label: "DMS 可视化", project: "dms", kind: "visualize" },
{ id: "promote_dms", label: "DMS 晋级", project: "dms", kind: "promote" },
{ id: "train_lane", label: "Lane 训练", project: "lane", kind: "train" },
{ id: "eval_lane", label: "Lane 评估", project: "lane", kind: "eval" },
...(LANE_DATA_VIZ_ENABLED
? [{ id: "visualize_lane" as const, label: "Lane 可视化", project: "lane" as const, kind: "visualize" }]
: []),
];
function statusBadge(status: string): string {
if (status === "succeeded") return "badge-promoted";
if (status === "running") return "badge-training";
if (status === "failed") return "badge-pending";
if (status === "queued") return "badge-pending";
return "badge-idle";
}
function kindLabel(kind: string): string {
return (
{
train: "训练",
eval: "评估",
visualize: "可视化",
promote: "晋级",
pipeline: "流水线",
}[kind] || kind
);
}
function fmtTime(ts?: string | null): string {
if (!ts) return "—";
const d = new Date(ts);
if (Number.isNaN(d.getTime())) return ts;
return d.toLocaleString("zh-CN", { hour12: false });
}
function fmtDuration(sec?: number | null): string {
if (sec == null) return "—";
if (sec < 60) return `${sec.toFixed(1)}s`;
const m = Math.floor(sec / 60);
const s = Math.round(sec % 60);
return `${m}m ${s}s`;
}
function fmtMetric(metrics: Record<string, unknown>): string {
const parts: string[] = [];
if (metrics.map50 != null) parts.push(`mAP50=${metrics.map50}`);
if (metrics.delta_map50 != null) parts.push(`Δ=${metrics.delta_map50}`);
return parts.length > 0 ? parts.join(" · ") : "—";
}
export function TrainingPage() {
const { refreshMeta } = useOutletContext<Ctx>();
const toast = useToast();
const [pending, setPending] = useState<PendingReport | null>(null);
const [records, setRecords] = useState<TrainingRecord[]>([]);
const [summary, setSummary] = useState({ total: 0, running: 0, queued: 0, succeeded: 0, failed: 0 });
const [models, setModels] = useState<ModelRegistry | null>(null);
const [selectedId, setSelectedId] = useState<string | null>(null);
const [loading, setLoading] = useState(true);
const [filterProject, setFilterProject] = useState<"" | "dms" | "lane">("");
const [filterKind, setFilterKind] = useState("");
const [filterStatus, setFilterStatus] = useState("");
const [filterTask, setFilterTask] = useState("");
const [createAction, setCreateAction] = useState<TrainAction>("train_dms");
const [createNote, setCreateNote] = useState("");
const [dmsTask, setDmsTask] = useState("dam");
const [dmsTrack, setDmsTrack] = useState<"platform" | "local">("platform");
const [dmsWeights, setDmsWeights] = useState("");
const [laneTrack, setLaneTrack] = useState<"platform" | "local">("platform");
const [laneModelPath, setLaneModelPath] = useState("");
const [laneDataRoot, setLaneDataRoot] = useState("");
const [laneTestList, setLaneTestList] = useState("list/test_gt.txt");
const loadAll = useCallback(async () => {
setLoading(true);
try {
const [p, list, reg] = await Promise.all([
api.pending(),
api.listTrainingRecords({
project: filterProject || undefined,
kind: filterKind || undefined,
status: filterStatus || undefined,
task: filterTask || undefined,
limit: 200,
}),
api.getModelRegistry("dms", filterTask || undefined),
]);
setPending(p);
setRecords(list.items || []);
setSummary(list.summary || { total: 0, running: 0, queued: 0, succeeded: 0, failed: 0 });
setModels(reg);
} catch (e) {
toast(String(e), true);
} finally {
setLoading(false);
}
}, [filterProject, filterKind, filterStatus, filterTask, toast]);
useEffect(() => {
loadAll();
}, [loadAll]);
useEffect(() => {
const tasks = Object.keys(pending?.projects?.dms?.task_defs || {});
if (tasks.length > 0 && !tasks.includes(dmsTask)) {
setDmsTask(tasks[0]);
}
}, [pending, dmsTask]);
useEffect(() => {
const hasRunning = records.some((r) => r.status === "running" || r.status === "queued");
if (!hasRunning) return;
const t = setInterval(loadAll, 5000);
return () => clearInterval(t);
}, [records, loadAll]);
const selected = useMemo(
() => records.find((r) => r.id === selectedId) || records[0] || null,
[records, selectedId]
);
useEffect(() => {
if (!selectedId && records.length > 0) {
setSelectedId(records[0].id);
}
}, [records, selectedId]);
const dmsTasks = Object.keys(pending?.projects?.dms?.task_defs || {});
const createMeta = CREATE_ACTIONS.find((a) => a.id === createAction);
const buildCreateParams = (): Record<string, unknown> => {
switch (createAction) {
case "train_dms":
return { task: dmsTask, mode: "full", track: dmsTrack };
case "eval_dms":
return { task: dmsTask, ...(dmsWeights ? { weights: dmsWeights } : {}) };
case "visualize_dms":
return { task: dmsTask, ...(dmsWeights ? { weights: dmsWeights } : {}) };
case "promote_dms":
return { task: dmsTask };
case "train_lane":
return { track: laneTrack };
case "eval_lane":
return {
model_path: laneModelPath,
test_list: laneTestList,
...(laneDataRoot ? { data_root: laneDataRoot } : {}),
};
case "visualize_lane":
return {
model_path: laneModelPath,
test_list: laneTestList,
...(laneDataRoot ? { data_root: laneDataRoot } : {}),
};
default:
return {};
}
};
const handleCreate = async () => {
if (createAction === "eval_lane" && !laneModelPath.trim()) {
toast("Lane 评估需填写模型路径", true);
return;
}
try {
const approval = await api.createTrainingRecord(createAction, buildCreateParams(), createNote || undefined);
toast(`已提交审核单 ${approval.id}`);
refreshMeta();
await loadAll();
setCreateNote("");
} catch (e) {
toast(String(e), true);
}
};
const currentTaskVersion =
filterTask && models?.tasks?.[filterTask]
? models.tasks[filterTask]
: dmsTask && models?.tasks?.[dmsTask]
? models.tasks[dmsTask]
: null;
return (
<>
<div className="kpi-strip">
<div className="kpi">
<span className="kpi-val">{summary.total}</span>
<span className="kpi-lbl"></span>
</div>
<div className="kpi">
<span className="kpi-val">{summary.running + summary.queued}</span>
<span className="kpi-lbl"></span>
</div>
<div className="kpi">
<span className="kpi-val">{summary.succeeded}</span>
<span className="kpi-lbl"></span>
</div>
<div className="kpi">
<span className="kpi-val">{summary.failed}</span>
<span className="kpi-lbl"></span>
</div>
<div className="kpi">
<span className="kpi-val text-sm mono">{currentTaskVersion?.current ? "有线上" : "—"}</span>
<span className="kpi-lbl">DMS </span>
</div>
</div>
<div className="panel">
<div className="panel-header">
<h2> / </h2>
</div>
<div className="panel-body">
<div className="crud-form">
<label className="field">
<span></span>
<select value={createAction} onChange={(e) => setCreateAction(e.target.value as TrainAction)}>
{CREATE_ACTIONS.map((a) => (
<option key={a.id} value={a.id}>
{a.label}
</option>
))}
</select>
</label>
<label className="field">
<span></span>
<input
value={createNote}
onChange={(e) => setCreateNote(e.target.value)}
placeholder="可选:版本说明、实验目的"
/>
</label>
{createMeta?.project === "dms" && (
<>
<label className="field">
<span></span>
<select value={dmsTask} onChange={(e) => setDmsTask(e.target.value)}>
{(dmsTasks.length > 0 ? dmsTasks : [dmsTask]).map((t) => (
<option key={t} value={t}>
{t}
</option>
))}
</select>
</label>
{createAction === "train_dms" && (
<label className="field">
<span></span>
<select value={dmsTrack} onChange={(e) => setDmsTrack(e.target.value as "platform" | "local")}>
<option value="platform">platform</option>
<option value="local">local</option>
</select>
</label>
)}
{["eval_dms", "visualize_dms"].includes(createAction) && (
<label className="field" style={{ gridColumn: "1 / -1" }}>
<span></span>
<input
value={dmsWeights}
onChange={(e) => setDmsWeights(e.target.value)}
placeholder="/path/to/best.pt"
/>
</label>
)}
</>
)}
{createMeta?.project === "lane" && (
<>
{createAction === "train_lane" && (
<label className="field">
<span></span>
<select value={laneTrack} onChange={(e) => setLaneTrack(e.target.value as "platform" | "local")}>
<option value="platform">platform</option>
<option value="local">local</option>
</select>
</label>
)}
{createAction !== "train_lane" && (
<>
<label className="field" style={{ gridColumn: "1 / -1" }}>
<span></span>
<input
value={laneModelPath}
onChange={(e) => setLaneModelPath(e.target.value)}
placeholder="/path/to/best.pth"
/>
</label>
<label className="field">
<span>test_list</span>
<input value={laneTestList} onChange={(e) => setLaneTestList(e.target.value)} />
</label>
<label className="field">
<span>data_root</span>
<input value={laneDataRoot} onChange={(e) => setLaneDataRoot(e.target.value)} />
</label>
</>
)}
</>
)}
<div className="crud-actions">
<button type="button" className="btn btn-primary" onClick={handleCreate}>
</button>
<span className="text-dim"></span>
</div>
</div>
</div>
</div>
<div className="panel">
<div className="panel-header">
<h2></h2>
<button type="button" className="btn btn-sm btn-ghost" onClick={() => loadAll()}>
</button>
</div>
<div className="panel-body">
<div className="catalog-toolbar">
<label className="field">
<span></span>
<select value={filterProject} onChange={(e) => setFilterProject(e.target.value as "" | "dms" | "lane")}>
<option value=""></option>
<option value="dms">DMS</option>
<option value="lane">Lane</option>
</select>
</label>
<label className="field">
<span></span>
<select value={filterKind} onChange={(e) => setFilterKind(e.target.value)}>
<option value=""></option>
<option value="train"></option>
<option value="eval"></option>
<option value="visualize"></option>
<option value="promote"></option>
</select>
</label>
<label className="field">
<span></span>
<select value={filterStatus} onChange={(e) => setFilterStatus(e.target.value)}>
<option value=""></option>
<option value="queued">queued</option>
<option value="running">running</option>
<option value="succeeded">succeeded</option>
<option value="failed">failed</option>
</select>
</label>
<label className="field">
<span>DMS </span>
<select value={filterTask} onChange={(e) => setFilterTask(e.target.value)}>
<option value=""></option>
{dmsTasks.map((t) => (
<option key={t} value={t}>
{t}
</option>
))}
</select>
</label>
</div>
{loading && records.length === 0 ? (
<p className="empty-state"></p>
) : records.length === 0 ? (
<p className="empty-state"></p>
) : (
<div className="grid-2-equal">
<div className="panel panel-compact">
<div className="panel-header">
<h2></h2>
<span className="text-dim">{records.length} </span>
</div>
<div className="panel-body table-wrap">
<table className="data-table compact">
<thead>
<tr>
<th></th>
<th></th>
<th></th>
<th></th>
<th></th>
</tr>
</thead>
<tbody>
{records.map((r) => (
<tr
key={r.id}
className={selected?.id === r.id ? "row-active" : ""}
style={{ cursor: "pointer" }}
onClick={() => setSelectedId(r.id)}
>
<td className="text-sm">{fmtTime(r.started_at || r.created_at).slice(0, 16)}</td>
<td>
<div>{r.action_label || r.action}</div>
<div className="text-dim">
{r.project} · {kindLabel(r.kind || "")}
</div>
</td>
<td>{r.task || "—"}</td>
<td>
<span className={`badge ${statusBadge(r.status)}`}>{r.status}</span>
</td>
<td className="text-sm">{fmtMetric(r.metrics || {})}</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
<div className="panel panel-compact">
<div className="panel-header">
<h2></h2>
</div>
<div className="panel-body">
{selected ? (
<TrainingDetail record={selected} />
) : (
<p className="empty-state"></p>
)}
</div>
</div>
</div>
)}
</div>
</div>
{models && Object.keys(models.tasks || {}).length > 0 && (
<div className="panel">
<div className="panel-header">
<h2>DMS </h2>
</div>
<div className="panel-body table-wrap">
<table className="data-table compact">
<thead>
<tr>
<th></th>
<th> (candidate)</th>
<th>线 (current)</th>
<th></th>
</tr>
</thead>
<tbody>
{Object.entries(models.tasks || {}).map(([task, ver]) => {
const v = ver as Record<string, unknown>;
const lastEval = (v.last_eval || {}) as Record<string, unknown>;
return (
<tr key={task}>
<td>
<strong>{task}</strong>
</td>
<td className="mono text-sm path-dd">{String(v.candidate || "—")}</td>
<td className="mono text-sm path-dd">{String(v.current || "—")}</td>
<td className="text-sm">
{lastEval.map50 != null ? `mAP50=${lastEval.map50}` : "—"}
{lastEval.weights ? (
<div className="text-dim mono">{String(lastEval.weights)}</div>
) : null}
</td>
</tr>
);
})}
</tbody>
</table>
</div>
</div>
)}
{(models?.eval_history?.length || 0) > 0 && (
<div className="panel">
<div className="panel-header">
<h2></h2>
</div>
<div className="panel-body table-wrap">
<table className="data-table compact">
<thead>
<tr>
<th></th>
<th></th>
<th>mAP50</th>
<th>Δ</th>
<th></th>
</tr>
</thead>
<tbody>
{(models?.eval_history || []).map((row, i) => (
<tr key={`${row.ts}-${i}`}>
<td className="text-sm">{fmtTime(row.ts as string)}</td>
<td>{String(row.task || "—")}</td>
<td>{row.map50 != null ? String(row.map50) : "—"}</td>
<td>{row.delta_map50 != null ? String(row.delta_map50) : "—"}</td>
<td className="mono text-sm path-dd">{String(row.weights || "—")}</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
)}
</>
);
}
function TrainingDetail({ record }: { record: TrainingRecord }) {
const approval = record.approval;
return (
<div className="batch-detail">
<div className="batch-detail-header">
<h3>{record.action_label || record.action}</h3>
<span className={`badge ${statusBadge(record.status)}`}>{record.status}</span>
</div>
<dl className="detail-dl">
<dt>Job ID</dt>
<dd className="mono">{record.id}</dd>
<dt> / </dt>
<dd>
{record.project} · {kindLabel(record.kind || "")}
{record.task ? ` · ${record.task}` : ""}
{record.track ? ` · ${record.track}` : ""}
</dd>
<dt> / / </dt>
<dd>
{fmtTime(record.created_at)} {fmtTime(record.started_at)} {fmtTime(record.finished_at)}
</dd>
<dt></dt>
<dd>{fmtDuration(record.duration_sec)}</dd>
<dt> / </dt>
<dd className="path-dd mono">{record.weight_path || "—"}</dd>
<dt></dt>
<dd>{fmtMetric(record.metrics || {})}</dd>
{record.error && (
<>
<dt></dt>
<dd className="text-sm" style={{ color: "var(--danger)" }}>
{record.error}
</dd>
</>
)}
{approval && (
<>
<dt></dt>
<dd>
<Link to={`/audit/${approval.id}`}>{approval.id}</Link>
<span className="text-dim"> · {approval.status}</span>
{approval.note ? <div className="text-sm">{approval.note}</div> : null}
</dd>
</>
)}
<dt></dt>
<dd>
<pre className="params-pre">{JSON.stringify(record.params || {}, null, 2)}</pre>
</dd>
<dt></dt>
<dd>
<pre className="params-pre milestone-result">{JSON.stringify(record.result || {}, null, 2)}</pre>
</dd>
</dl>
</div>
);
}

View File

@@ -0,0 +1,163 @@
import { useEffect, useMemo, useState } from "react";
import { api, type DataCandidate } from "../api/client";
import { useToast } from "../components/Toast";
export function UploadsPage({ embedded = false }: { embedded?: boolean }) {
const toast = useToast();
const [project, setProject] = useState<"dms" | "lane">("dms");
const [task, setTask] = useState("dam");
const [tasks, setTasks] = useState<string[]>([]);
const [file, setFile] = useState<File | null>(null);
const [progress, setProgress] = useState(0);
const [uploading, setUploading] = useState(false);
const [candidates, setCandidates] = useState<DataCandidate[]>([]);
const [activeId, setActiveId] = useState<string | null>(null);
const loadCandidates = async () => {
const res = await api.listDataCandidates(80);
setCandidates(res.items || []);
};
useEffect(() => {
loadCandidates();
}, []);
useEffect(() => {
const loadTasks = async () => {
try {
const pending = await api.pending();
const defs = pending.projects?.dms?.task_defs || {};
const names = Object.keys(defs);
setTasks(names);
if (names.length > 0 && !names.includes(task)) {
setTask(names[0]);
}
} catch {
// keep manual fallback value
}
};
loadTasks();
}, []);
useEffect(() => {
if (!activeId) return;
const t = setInterval(async () => {
try {
const latest = await api.getDataCandidate(activeId);
setCandidates((prev) => [latest, ...prev.filter((x) => x.id !== latest.id)]);
if (latest.status === "analyzed" || latest.status === "failed") {
setActiveId(null);
await loadCandidates();
}
} catch {
// keep polling loop running; backend errors are surfaced in status list
}
}, 2000);
return () => clearInterval(t);
}, [activeId]);
const submit = async (e: React.FormEvent) => {
e.preventDefault();
if (!file) {
toast("请先选择压缩包文件");
return;
}
setUploading(true);
setProgress(0);
try {
const res = await api.uploadDatasetFile(file, project, project === "dms" ? task : undefined, setProgress);
setActiveId(res.candidate.id);
setCandidates((prev) => [res.candidate, ...prev.filter((x) => x.id !== res.candidate.id)]);
toast(`上传成功,开始分析:${res.candidate.id}`);
} catch (err) {
toast(err instanceof Error ? err.message : "上传失败");
} finally {
setUploading(false);
}
};
const active = useMemo(() => candidates.find((x) => x.id === activeId) || null, [candidates, activeId]);
return (
<>
<div className={embedded ? "" : "panel"}>
<div className="panel-header"><h2></h2></div>
<div className="panel-body">
<form className="crud-form" onSubmit={submit}>
<label className="field">
<span>Project</span>
<select value={project} onChange={(e) => {
const next = e.target.value as "dms" | "lane";
setProject(next);
if (next === "dms" && tasks.length > 0 && !tasks.includes(task)) {
setTask(tasks[0]);
}
}}>
<option value="dms">dms</option>
<option value="lane">lane</option>
</select>
</label>
{project === "dms" && (
<label className="field">
<span>Task</span>
<select value={task} onChange={(e) => setTask(e.target.value)}>
{(tasks.length > 0 ? tasks : [task]).map((t) => (
<option key={t} value={t}>{t}</option>
))}
</select>
</label>
)}
<label className="field" style={{ gridColumn: "1 / -1" }}>
<span></span>
<input type="file" accept=".zip,.tar,.tar.gz,.tgz" onChange={(e) => setFile(e.target.files?.[0] || null)} />
</label>
<div className="crud-actions">
<button type="submit" className="btn btn-primary" disabled={uploading}>
{uploading ? "上传中..." : "上传并自动分析"}
</button>
<button type="button" className="btn btn-ghost" onClick={() => loadCandidates()}></button>
</div>
</form>
<div style={{ marginTop: 12 }}>
<div className="progress-bar"><div className="progress-fill" style={{ width: `${progress}%` }} /></div>
<p className="audit-note">{progress}% {active ? `| 当前分析状态:${active.status}` : ""}</p>
</div>
</div>
</div>
<div className={embedded ? "panel panel-compact" : "panel"}>
<div className="panel-header"><h2></h2></div>
<div className="panel-body table-wrap">
<table className="data-table compact">
<thead>
<tr>
<th>ID</th>
<th>/</th>
<th></th>
<th></th>
<th>train/val/test</th>
<th></th>
</tr>
</thead>
<tbody>
{candidates.map((c) => (
<tr key={c.id} className={c.id === activeId ? "row-active" : ""}>
<td className="mono">{c.id}</td>
<td>{c.project}{c.task ? `/${c.task}` : ""}</td>
<td><span className="badge badge-idle">{c.status}</span></td>
<td>{c.format_id || "—"}</td>
<td>
{c.split_counts
? `${c.split_counts.train || 0}/${c.split_counts.val || 0}/${c.split_counts.test || 0}`
: "—"}
</td>
<td>{c.error_message || "—"}</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
</>
);
}

File diff suppressed because it is too large Load Diff

9
platform/web/src/vite-env.d.ts vendored Normal file
View File

@@ -0,0 +1,9 @@
/// <reference types="vite/client" />
interface ImportMetaEnv {
readonly VITE_API_BASE: string;
}
interface ImportMeta {
readonly env: ImportMetaEnv;
}

View File

@@ -0,0 +1,21 @@
{
"compilerOptions": {
"target": "ES2022",
"useDefineForClassFields": true,
"lib": ["ES2022", "DOM", "DOM.Iterable"],
"module": "ESNext",
"skipLibCheck": true,
"moduleResolution": "bundler",
"allowImportingTsExtensions": true,
"isolatedModules": true,
"moduleDetection": "force",
"noEmit": true,
"jsx": "react-jsx",
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noFallthroughCasesInSwitch": true,
"noUncheckedSideEffectImports": true
},
"include": ["src"]
}

View File

@@ -0,0 +1,22 @@
import { defineConfig, loadEnv } from "vite";
import react from "@vitejs/plugin-react";
export default defineConfig(({ mode }) => {
const env = loadEnv(mode, process.cwd(), "");
const proxyTarget = env.VITE_API_PROXY || "http://127.0.0.1:8787";
return {
plugins: [react()],
server: {
host: true,
port: 5173,
proxy: {
"/api": { target: proxyTarget, changeOrigin: true },
},
},
build: {
outDir: "dist",
emptyOutDir: true,
},
};
});