feat: initial HSAP platform
Huaxu Sentinel Active Safety Platform with embedded algorithm code, Docker Compose setup, and vendored dataset scaffolds for clone-and-run. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
4
platform/as_platform/__init__.py
Normal file
4
platform/as_platform/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
"""华胥智能主动安全算法迭代平台。"""
|
||||
from as_platform import sdk # noqa: F401
|
||||
|
||||
__all__ = ["sdk"]
|
||||
0
platform/as_platform/agents/__init__.py
Normal file
0
platform/as_platform/agents/__init__.py
Normal file
0
platform/as_platform/agents/graphs/__init__.py
Normal file
0
platform/as_platform/agents/graphs/__init__.py
Normal file
30
platform/as_platform/agents/graphs/ingest_flow.py
Normal file
30
platform/as_platform/agents/graphs/ingest_flow.py
Normal file
@@ -0,0 +1,30 @@
|
||||
"""ingest_flow:感知 returned 批次 → 提交 build 审核。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from as_platform.agents.tools import invoke_tool, submit_build_for_batch
|
||||
from as_platform.agents.trace import start_trace, trace_span
|
||||
|
||||
|
||||
def run_ingest_flow(*, task: str = "dam", submitted_by: str = "agent") -> dict[str, Any]:
|
||||
trace_id = start_trace("ingest_flow", task=task)
|
||||
with trace_span("list_pending"):
|
||||
report = invoke_tool("list_pending_batches")
|
||||
|
||||
submitted = []
|
||||
for batch in report.get("batches", []):
|
||||
if batch.get("task") != task:
|
||||
continue
|
||||
if batch.get("stage") != "returned":
|
||||
continue
|
||||
with trace_span("submit_build", batch=batch.get("batch")):
|
||||
apr = submit_build_for_batch(
|
||||
task=task,
|
||||
batch=batch["batch"],
|
||||
pack=batch.get("pack") or "dms_v2",
|
||||
submitted_by=submitted_by,
|
||||
)
|
||||
submitted.append(apr)
|
||||
|
||||
return {"trace_id": trace_id, "submitted": submitted, "count": len(submitted)}
|
||||
20
platform/as_platform/agents/graphs/labeling_flow.py
Normal file
20
platform/as_platform/agents/graphs/labeling_flow.py
Normal file
@@ -0,0 +1,20 @@
|
||||
"""labeling_flow:列出 raw_pool / out_for_labeling 批次。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from as_platform.agents.tools import invoke_tool
|
||||
from as_platform.agents.trace import start_trace, trace_span
|
||||
|
||||
|
||||
def run_labeling_flow(*, task: str | None = None) -> dict[str, Any]:
|
||||
trace_id = start_trace("labeling_flow", task=task)
|
||||
with trace_span("list_pending"):
|
||||
report = invoke_tool("list_pending_batches")
|
||||
|
||||
batches = [
|
||||
b for b in report.get("batches", [])
|
||||
if b.get("stage") in ("raw_pool", "out_for_labeling", "returned")
|
||||
and (task is None or b.get("task") == task)
|
||||
]
|
||||
return {"trace_id": trace_id, "batches": batches, "count": len(batches)}
|
||||
18
platform/as_platform/agents/graphs/train_promote_flow.py
Normal file
18
platform/as_platform/agents/graphs/train_promote_flow.py
Normal file
@@ -0,0 +1,18 @@
|
||||
"""train_promote_flow:提交 train 审核(platform 轨)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from as_platform.agents.tools import get_model_versions, invoke_tool, submit_train_job
|
||||
from as_platform.agents.trace import start_trace, trace_span
|
||||
|
||||
|
||||
def run_train_promote_flow(*, task: str = "dam", submitted_by: str = "agent") -> dict[str, Any]:
|
||||
trace_id = start_trace("train_promote_flow", task=task)
|
||||
with trace_span("get_versions"):
|
||||
versions = get_model_versions(task)
|
||||
|
||||
with trace_span("submit_train"):
|
||||
apr = submit_train_job("dms", task, track="platform", submitted_by=submitted_by)
|
||||
|
||||
return {"trace_id": trace_id, "versions_before": versions, "approval": apr}
|
||||
86
platform/as_platform/agents/tools.py
Normal file
86
platform/as_platform/agents/tools.py
Normal file
@@ -0,0 +1,86 @@
|
||||
"""LangChain 风格 Tool 注册(纯 Python + 可选 langchain)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
from as_platform.audit.queue import submit_approval
|
||||
from as_platform.data.core import get_catalog, get_pending_report, load_wf
|
||||
from as_platform.jobs.queue import get_job, list_jobs
|
||||
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
|
||||
WORKSPACE = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def list_pending_batches() -> dict[str, Any]:
|
||||
return get_pending_report()
|
||||
|
||||
|
||||
def get_dataset_catalog() -> dict[str, Any]:
|
||||
return get_catalog()
|
||||
|
||||
|
||||
def submit_build_for_batch(task: str, batch: str, pack: str = "dms_v2", submitted_by: str | None = None) -> dict:
|
||||
return submit_approval(
|
||||
"build_dms",
|
||||
{"task": task, "pack": pack, "batch": batch},
|
||||
submitted_by=submitted_by,
|
||||
note=f"agent build {batch}",
|
||||
)
|
||||
|
||||
|
||||
def submit_train_job(project: str, task: str, track: str = "platform", submitted_by: str | None = None) -> dict:
|
||||
action = "train_dms" if project == "dms" else "train_lane"
|
||||
params: dict[str, Any] = {"track": track}
|
||||
if project == "dms":
|
||||
params["task"] = task
|
||||
return submit_approval(action, params, submitted_by=submitted_by, note=f"agent train {project}/{task}")
|
||||
|
||||
|
||||
def get_job_status(job_id: str) -> dict[str, Any] | None:
|
||||
return get_job(job_id)
|
||||
|
||||
|
||||
def get_model_versions(task: str) -> dict[str, Any]:
|
||||
root = WORKSPACE / "datasets/dms/manifests/train_versions.yaml"
|
||||
if not root.is_file():
|
||||
return {}
|
||||
data = yaml.safe_load(root.read_text(encoding="utf-8"))
|
||||
return data.get(task, {})
|
||||
|
||||
|
||||
TOOL_REGISTRY: dict[str, Callable[..., Any]] = {
|
||||
"list_pending_batches": list_pending_batches,
|
||||
"get_dataset_catalog": get_dataset_catalog,
|
||||
"submit_build_for_batch": submit_build_for_batch,
|
||||
"submit_train_job": submit_train_job,
|
||||
"get_job_status": get_job_status,
|
||||
"get_model_versions": get_model_versions,
|
||||
}
|
||||
|
||||
|
||||
def invoke_tool(name: str, **kwargs: Any) -> Any:
|
||||
fn = TOOL_REGISTRY.get(name)
|
||||
if not fn:
|
||||
raise ValueError(f"未知 tool: {name}")
|
||||
return fn(**kwargs)
|
||||
|
||||
|
||||
def as_langchain_tools() -> list[Any]:
|
||||
try:
|
||||
from langchain_core.tools import tool
|
||||
except ImportError:
|
||||
return []
|
||||
|
||||
@tool
|
||||
def t_list_pending_batches() -> dict:
|
||||
"""列出待处理批次与送标状态。"""
|
||||
return list_pending_batches()
|
||||
|
||||
@tool
|
||||
def t_get_dataset_catalog() -> dict:
|
||||
"""获取 DMS/Lane 数据目录统计。"""
|
||||
return get_dataset_catalog()
|
||||
|
||||
return [t_list_pending_batches, t_get_dataset_catalog]
|
||||
62
platform/as_platform/agents/trace.py
Normal file
62
platform/as_platform/agents/trace.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""LangSmith 式 trace:manifests/trace_log.jsonl"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Iterator
|
||||
|
||||
from as_platform.config import TRACE_LOG, MANIFESTS
|
||||
|
||||
_current_trace: str | None = None
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def start_trace(name: str, **meta: Any) -> str:
|
||||
global _current_trace
|
||||
trace_id = f"trace-{uuid.uuid4().hex[:12]}"
|
||||
_current_trace = trace_id
|
||||
_append({"type": "trace_start", "trace_id": trace_id, "name": name, "ts": _now(), **meta})
|
||||
return trace_id
|
||||
|
||||
|
||||
def _append(entry: dict[str, Any]) -> None:
|
||||
MANIFESTS.mkdir(parents=True, exist_ok=True)
|
||||
if not TRACE_LOG.is_file():
|
||||
TRACE_LOG.write_text("", encoding="utf-8")
|
||||
with TRACE_LOG.open("a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def trace_span(span_type: str, **fields: Any) -> Iterator[None]:
|
||||
span_id = f"span-{uuid.uuid4().hex[:8]}"
|
||||
_append({"type": span_type, "span_id": span_id, "trace_id": _current_trace, "ts": _now(), **fields})
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_append({"type": f"{span_type}_end", "span_id": span_id, "trace_id": _current_trace, "ts": _now()})
|
||||
|
||||
|
||||
def get_trace(trace_id: str) -> list[dict[str, Any]]:
|
||||
if not TRACE_LOG.is_file():
|
||||
return []
|
||||
return [json.loads(l) for l in TRACE_LOG.read_text().strip().splitlines() if l.strip() and trace_id in l]
|
||||
|
||||
|
||||
def list_traces(limit: int = 50) -> list[str]:
|
||||
if not TRACE_LOG.is_file():
|
||||
return []
|
||||
ids = []
|
||||
for line in TRACE_LOG.read_text().strip().splitlines():
|
||||
try:
|
||||
o = json.loads(line)
|
||||
if o.get("type") == "trace_start" and o.get("trace_id"):
|
||||
ids.append(o["trace_id"])
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return list(reversed(ids[-limit:]))
|
||||
4
platform/as_platform/api/__init__.py
Normal file
4
platform/as_platform/api/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
"""API 入口。"""
|
||||
from as_platform.api.server import app, main
|
||||
|
||||
__all__ = ["app", "main"]
|
||||
14
platform/as_platform/api/__main__.py
Normal file
14
platform/as_platform/api/__main__.py
Normal file
@@ -0,0 +1,14 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_ROOT = Path(__file__).resolve().parents[3] # HSAP repo root
|
||||
_PLATFORM = _ROOT / "platform"
|
||||
for p in (_ROOT, _PLATFORM):
|
||||
s = str(p)
|
||||
if s not in sys.path:
|
||||
sys.path.insert(0, s)
|
||||
|
||||
from as_platform.api.server import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
98
platform/as_platform/api/auth_routes.py
Normal file
98
platform/as_platform/api/auth_routes.py
Normal file
@@ -0,0 +1,98 @@
|
||||
"""认证与用户管理 API。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import RedirectResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from as_platform.auth.deps import get_current_user, require_permission
|
||||
from as_platform.auth.feishu import build_authorize_url, exchange_code, is_feishu_configured, verify_state
|
||||
from as_platform.auth.jwt import create_access_token
|
||||
from as_platform.auth.users import get_or_create_dev_user, list_users, set_user_roles, upsert_feishu_user
|
||||
from as_platform.config import DEV_AUTH_ENABLED, FRONTEND_URL
|
||||
from as_platform.db.engine import get_db
|
||||
from as_platform.db.init_db import user_to_dict
|
||||
from as_platform.db.models import User
|
||||
|
||||
router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
|
||||
|
||||
|
||||
class DevLoginBody(BaseModel):
|
||||
name: str = "开发用户"
|
||||
|
||||
|
||||
class SetRolesBody(BaseModel):
|
||||
role_codes: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
@router.get("/config")
|
||||
def auth_config() -> dict[str, Any]:
|
||||
return {
|
||||
"feishu_enabled": is_feishu_configured(),
|
||||
"dev_auth_enabled": DEV_AUTH_ENABLED and not is_feishu_configured(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/feishu/authorize")
|
||||
def feishu_authorize():
|
||||
if not is_feishu_configured():
|
||||
raise HTTPException(503, "未配置飞书应用,请设置 FEISHU_APP_ID / FEISHU_APP_SECRET")
|
||||
url, _ = build_authorize_url()
|
||||
return RedirectResponse(url)
|
||||
|
||||
|
||||
@router.get("/feishu/callback")
|
||||
def feishu_callback(code: str, state: str, db: Annotated[Session, Depends(get_db)]):
|
||||
if not verify_state(state):
|
||||
raise HTTPException(400, "无效的 state,请重新登录")
|
||||
try:
|
||||
info = exchange_code(code)
|
||||
except Exception as e:
|
||||
raise HTTPException(502, f"飞书登录失败: {e}") from e
|
||||
user = upsert_feishu_user(db, info)
|
||||
db.commit()
|
||||
token = create_access_token(user.id)
|
||||
# 避免 BrowserRouter 直达路径 404,统一回根路径并附带 token
|
||||
qs = urlencode({"token": token})
|
||||
return RedirectResponse(f"{FRONTEND_URL}/?{qs}")
|
||||
|
||||
|
||||
@router.post("/dev/login")
|
||||
def dev_login(body: DevLoginBody, db: Annotated[Session, Depends(get_db)]):
|
||||
if not DEV_AUTH_ENABLED or is_feishu_configured():
|
||||
raise HTTPException(403, "开发登录未启用")
|
||||
user = get_or_create_dev_user(db, body.name)
|
||||
db.commit()
|
||||
token = create_access_token(user.id)
|
||||
return {"access_token": token, "user": user_to_dict(user)}
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
def auth_me(user: Annotated[User, Depends(get_current_user)]):
|
||||
return user_to_dict(user)
|
||||
|
||||
|
||||
@router.get("/users")
|
||||
def auth_list_users(
|
||||
_user: Annotated[User, Depends(require_permission("admin:users"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
):
|
||||
return {"items": [user_to_dict(u) for u in list_users(db)]}
|
||||
|
||||
|
||||
@router.put("/users/{user_id}/roles")
|
||||
def auth_set_roles(
|
||||
user_id: int,
|
||||
body: SetRolesBody,
|
||||
_user: Annotated[User, Depends(require_permission("admin:users"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
):
|
||||
user = set_user_roles(db, user_id, body.role_codes)
|
||||
if not user:
|
||||
raise HTTPException(404, "用户不存在")
|
||||
db.commit()
|
||||
return user_to_dict(user)
|
||||
584
platform/as_platform/api/server.py
Normal file
584
platform/as_platform/api/server.py
Normal file
@@ -0,0 +1,584 @@
|
||||
#!/usr/bin/env python3
|
||||
"""统一 API + React Web。cd HSAP && PYTHONPATH=platform python -m as_platform.api.server"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any
|
||||
import threading
|
||||
|
||||
try:
|
||||
from fastapi import Depends, FastAPI, File, Form, HTTPException, Query, UploadFile
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel, Field
|
||||
except ImportError as e:
|
||||
raise SystemExit("需要安装: pip install fastapi uvicorn pydantic sqlalchemy python-jose httpx") from e
|
||||
|
||||
from as_platform.agents.graphs.ingest_flow import run_ingest_flow
|
||||
from as_platform.agents.graphs.labeling_flow import run_labeling_flow
|
||||
from as_platform.agents.graphs.train_promote_flow import run_train_promote_flow
|
||||
from as_platform.agents.tools import TOOL_REGISTRY, invoke_tool
|
||||
from as_platform.agents.trace import get_trace, list_traces
|
||||
from as_platform.api.auth_routes import router as auth_router
|
||||
from as_platform.audit.queue import (
|
||||
ACTION_LABELS,
|
||||
ACTIONS_REQUIRING_APPROVAL,
|
||||
approve_and_execute,
|
||||
get_approval,
|
||||
list_approvals,
|
||||
reject_approval,
|
||||
submit_approval,
|
||||
)
|
||||
from as_platform.audit.preview import find_image_ref, list_scope_images, render_overlay, resolve_approval_scope
|
||||
from as_platform.auth.deps import can_submit_action, get_current_user, require_any_permission, require_permission
|
||||
from as_platform.config import IS_POSTGRES, PLATFORM_DIR, PLATFORM_WEB, WORKSPACE
|
||||
from as_platform.data.core import get_catalog, get_pending_report, register_batch, warmup_catalog_cache
|
||||
from as_platform.data.ingest import UnknownFormatError, inspect_uploaded_dataset
|
||||
from as_platform.data.lake import (
|
||||
create_uploaded_candidate,
|
||||
get_candidate,
|
||||
link_candidate_analysis_job,
|
||||
list_candidates as list_data_candidates,
|
||||
write_candidate_upload,
|
||||
)
|
||||
from as_platform.data.organize import organize_batch
|
||||
from as_platform.db.engine import check_connection
|
||||
from as_platform.db.init_db import init_database
|
||||
from as_platform.db.models import User
|
||||
from as_platform.jobs.queue import enqueue_job, get_job, list_jobs
|
||||
from as_platform.redis.bus import ping_redis
|
||||
from as_platform.training.service import (
|
||||
TRAINING_ACTIONS,
|
||||
create_training_submission,
|
||||
get_model_registry,
|
||||
get_training_record,
|
||||
list_training_records,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
init_database()
|
||||
threading.Thread(target=warmup_catalog_cache, daemon=True, name="catalog-warmup").start()
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(title="华胥智能主动安全平台", version="1.1.0", lifespan=lifespan)
|
||||
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
|
||||
app.include_router(auth_router)
|
||||
_UI_DIR: Path | None = None
|
||||
|
||||
|
||||
class SubmitApprovalBody(BaseModel):
|
||||
action: str
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
submitted_by: str | None = None
|
||||
note: str | None = None
|
||||
|
||||
|
||||
class ReviewBody(BaseModel):
|
||||
reviewed_by: str | None = None
|
||||
comment: str | None = None
|
||||
|
||||
|
||||
class RegisterBatchBody(BaseModel):
|
||||
project: str
|
||||
task: str | None = None
|
||||
batch: str
|
||||
pack: str | None = None
|
||||
stage: str = "returned"
|
||||
engineer: str | None = None
|
||||
location: str = "inbox"
|
||||
submitted_by: str | None = None
|
||||
skip_audit: bool = False
|
||||
|
||||
|
||||
class BuildFromBatchBody(BaseModel):
|
||||
project: str = "dms"
|
||||
task: str
|
||||
batch: str
|
||||
pack: str = "dms_v2"
|
||||
location: str = "inbox"
|
||||
submitted_by: str | None = None
|
||||
note: str | None = None
|
||||
|
||||
|
||||
class OrganizeBody(BaseModel):
|
||||
batch_path: str
|
||||
task: str | None = None
|
||||
|
||||
|
||||
class AgentInvokeBody(BaseModel):
|
||||
graph: str = "ingest_flow"
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ToolInvokeBody(BaseModel):
|
||||
tool: str
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class CreateTrainingBody(BaseModel):
|
||||
action: str
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
note: str | None = None
|
||||
|
||||
|
||||
class InspectUploadBody(BaseModel):
|
||||
project: str
|
||||
task: str | None = None
|
||||
source_path: str
|
||||
|
||||
|
||||
@app.get("/api/v1/health")
|
||||
def health() -> dict[str, str]:
|
||||
db_ok = check_connection()
|
||||
redis_ok = ping_redis()
|
||||
ok = db_ok and redis_ok
|
||||
return {
|
||||
"status": "ok" if ok else "degraded",
|
||||
"workspace": str(WORKSPACE),
|
||||
"database": "postgresql" if IS_POSTGRES else "sqlite",
|
||||
"db_connected": str(db_ok).lower(),
|
||||
"redis_connected": str(redis_ok).lower(),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/v1/pending")
|
||||
def api_pending(_user: Annotated[User, Depends(require_permission("read:pending"))]) -> dict[str, Any]:
|
||||
return get_pending_report()
|
||||
|
||||
|
||||
@app.get("/api/v1/catalog")
|
||||
def api_catalog(
|
||||
_user: Annotated[User, Depends(require_permission("read:catalog"))],
|
||||
refresh: bool = Query(False),
|
||||
) -> dict[str, Any]:
|
||||
return get_catalog(refresh=refresh)
|
||||
|
||||
|
||||
@app.get("/api/v1/catalog/dms/{task}")
|
||||
def api_catalog_dms(
|
||||
task: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:catalog"))],
|
||||
refresh: bool = Query(False),
|
||||
) -> dict[str, Any]:
|
||||
full = get_catalog(refresh=refresh)
|
||||
if task not in (full.get("dms") or {}):
|
||||
raise HTTPException(404, f"未知 DMS 任务: {task}")
|
||||
return {"task": task, **full["dms"][task]}
|
||||
|
||||
|
||||
@app.get("/api/v1/catalog/lane/{pack}")
|
||||
def api_catalog_lane(
|
||||
pack: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:catalog"))],
|
||||
refresh: bool = Query(False),
|
||||
) -> dict[str, Any]:
|
||||
full = get_catalog(refresh=refresh)
|
||||
if pack not in (full.get("lane") or {}):
|
||||
raise HTTPException(404, f"未知 Lane 包: {pack}")
|
||||
return {"pack": pack, **full["lane"][pack]}
|
||||
|
||||
|
||||
@app.get("/api/v1/actions")
|
||||
def api_actions(_user: Annotated[User, Depends(require_permission("read:audit"))]) -> dict[str, Any]:
|
||||
return {"actions": [{"id": k, "label": ACTION_LABELS.get(k, k)} for k in sorted(ACTIONS_REQUIRING_APPROVAL)]}
|
||||
|
||||
|
||||
@app.get("/api/v1/jobs")
|
||||
def api_jobs(
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
status: str | None = None,
|
||||
limit: int = Query(100, le=500),
|
||||
) -> dict[str, Any]:
|
||||
return {"items": list_jobs(status=status, limit=limit)}
|
||||
|
||||
|
||||
@app.get("/api/v1/jobs/{job_id}")
|
||||
def api_job(job_id: str, _user: Annotated[User, Depends(require_permission("read:jobs"))]) -> dict[str, Any]:
|
||||
job = get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(404, "Job 不存在")
|
||||
return job
|
||||
|
||||
|
||||
@app.get("/api/v1/training/actions")
|
||||
def api_training_actions(_user: Annotated[User, Depends(require_permission("read:jobs"))]) -> dict[str, Any]:
|
||||
return {
|
||||
"actions": [
|
||||
{"id": action, "label": ACTION_LABELS.get(action, action)}
|
||||
for action in sorted(TRAINING_ACTIONS)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/v1/training/records")
|
||||
def api_training_records(
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
project: str | None = None,
|
||||
kind: str | None = None,
|
||||
status: str | None = None,
|
||||
task: str | None = None,
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
) -> dict[str, Any]:
|
||||
return list_training_records(project=project, kind=kind, status=status, task=task, limit=limit)
|
||||
|
||||
|
||||
@app.get("/api/v1/training/records/{job_id}")
|
||||
def api_training_record(
|
||||
job_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
) -> dict[str, Any]:
|
||||
rec = get_training_record(job_id)
|
||||
if not rec:
|
||||
raise HTTPException(404, "训练记录不存在")
|
||||
return rec
|
||||
|
||||
|
||||
@app.get("/api/v1/training/models")
|
||||
def api_training_models(
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
project: str = Query("dms"),
|
||||
task: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return get_model_registry(project=project, task=task)
|
||||
|
||||
|
||||
@app.post("/api/v1/training/records")
|
||||
def api_create_training(
|
||||
body: CreateTrainingBody,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
) -> dict[str, Any]:
|
||||
if not can_submit_action(user, body.action):
|
||||
raise HTTPException(403, f"无权提交: {body.action}")
|
||||
try:
|
||||
return create_training_submission(
|
||||
body.action,
|
||||
body.params,
|
||||
submitted_by=user.name,
|
||||
submitted_by_user_id=user.id,
|
||||
note=body.note,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@app.get("/api/v1/traces")
|
||||
def api_traces(_user: Annotated[User, Depends(get_current_user)], limit: int = 50) -> dict[str, Any]:
|
||||
return {"trace_ids": list_traces(limit=limit)}
|
||||
|
||||
|
||||
@app.get("/api/v1/traces/{trace_id}")
|
||||
def api_trace(trace_id: str, _user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
|
||||
spans = get_trace(trace_id)
|
||||
if not spans:
|
||||
raise HTTPException(404, "Trace 不存在")
|
||||
return {"trace_id": trace_id, "spans": spans}
|
||||
|
||||
|
||||
@app.get("/api/v1/agents/tools")
|
||||
def api_tools(_user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
|
||||
return {"tools": list(TOOL_REGISTRY.keys())}
|
||||
|
||||
|
||||
@app.post("/api/v1/agents/tools/invoke")
|
||||
def api_tool_invoke(body: ToolInvokeBody, _user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
|
||||
try:
|
||||
return {"result": invoke_tool(body.tool, **body.params)}
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@app.post("/api/v1/agents/invoke")
|
||||
def api_agent_invoke(body: AgentInvokeBody, _user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
|
||||
graphs = {
|
||||
"ingest_flow": run_ingest_flow,
|
||||
"labeling_flow": run_labeling_flow,
|
||||
"train_promote_flow": run_train_promote_flow,
|
||||
}
|
||||
fn = graphs.get(body.graph)
|
||||
if not fn:
|
||||
raise HTTPException(400, f"未知 graph: {body.graph}")
|
||||
return fn(**body.params)
|
||||
|
||||
|
||||
@app.get("/api/v1/approvals")
|
||||
def api_list_approvals(
|
||||
_user: Annotated[User, Depends(require_permission("read:audit"))],
|
||||
status: str | None = None,
|
||||
limit: int = Query(100, le=500),
|
||||
) -> dict[str, Any]:
|
||||
return {"items": list_approvals(status=status, limit=limit)}
|
||||
|
||||
|
||||
@app.get("/api/v1/approvals/{record_id}")
|
||||
def api_get_approval(record_id: str, _user: Annotated[User, Depends(require_permission("read:audit"))]) -> dict[str, Any]:
|
||||
rec = get_approval(record_id)
|
||||
if not rec:
|
||||
raise HTTPException(404, "审核单不存在")
|
||||
return rec
|
||||
|
||||
|
||||
@app.get("/api/v1/approvals/{record_id}/preview")
|
||||
def api_approval_preview(
|
||||
record_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:audit"))],
|
||||
) -> dict[str, Any]:
|
||||
rec = get_approval(record_id)
|
||||
if not rec:
|
||||
raise HTTPException(404, "审核单不存在")
|
||||
try:
|
||||
scope = resolve_approval_scope(rec["action"], rec.get("params") or {})
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
batch_summaries = []
|
||||
for b in scope.get("batches") or []:
|
||||
batch_dir = Path(b["path"])
|
||||
batch_summaries.append(
|
||||
{
|
||||
"batch": b.get("batch"),
|
||||
"location": b.get("location"),
|
||||
"path": str(batch_dir) if batch_dir.is_dir() else None,
|
||||
"exists": batch_dir.is_dir(),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"approval": rec,
|
||||
"scope_label": scope.get("scope_label"),
|
||||
"task": scope.get("task"),
|
||||
"pack": scope.get("pack"),
|
||||
"class_names": scope.get("class_names"),
|
||||
"batches": batch_summaries,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/v1/approvals/{record_id}/images")
|
||||
def api_approval_images(
|
||||
record_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:audit"))],
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(60, ge=1, le=200),
|
||||
) -> dict[str, Any]:
|
||||
rec = get_approval(record_id)
|
||||
if not rec:
|
||||
raise HTTPException(404, "审核单不存在")
|
||||
try:
|
||||
scope = resolve_approval_scope(rec["action"], rec.get("params") or {})
|
||||
return list_scope_images(scope, offset=offset, limit=limit)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@app.get("/api/v1/approvals/{record_id}/images/{image_id}")
|
||||
def api_approval_image(
|
||||
record_id: str,
|
||||
image_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:audit"))],
|
||||
thumb: bool = Query(True),
|
||||
) -> Response:
|
||||
rec = get_approval(record_id)
|
||||
if not rec:
|
||||
raise HTTPException(404, "审核单不存在")
|
||||
try:
|
||||
scope = resolve_approval_scope(rec["action"], rec.get("params") or {})
|
||||
ref = find_image_ref(scope, image_id)
|
||||
if not ref or not ref.image_path.is_file():
|
||||
raise HTTPException(404, "图像不存在")
|
||||
class_names = scope.get("class_names") or {}
|
||||
max_size = 480 if thumb else 1920
|
||||
data = render_overlay(ref.image_path, ref.label_path, class_names, max_size=max_size)
|
||||
return Response(content=data, media_type="image/jpeg")
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@app.post("/api/v1/approvals/submit")
|
||||
def api_submit(body: SubmitApprovalBody, user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
|
||||
if not can_submit_action(user, body.action):
|
||||
raise HTTPException(403, f"无权提交: {body.action}")
|
||||
try:
|
||||
return submit_approval(
|
||||
body.action, body.params,
|
||||
submitted_by=user.name,
|
||||
submitted_by_user_id=user.id,
|
||||
note=body.note,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@app.post("/api/v1/approvals/submit-build-batch")
|
||||
def api_submit_build_batch(body: BuildFromBatchBody, user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
|
||||
if not can_submit_action(user, "build_dms"):
|
||||
raise HTTPException(403, "无权提交 build")
|
||||
params: dict[str, Any] = {"task": body.task, "pack": body.pack}
|
||||
if body.location == "inbox":
|
||||
params["batch"] = body.batch
|
||||
else:
|
||||
params["all_sources"] = True
|
||||
return submit_approval(
|
||||
"build_dms", params,
|
||||
submitted_by=user.name,
|
||||
submitted_by_user_id=user.id,
|
||||
note=body.note or f"入库 {body.batch}",
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/v1/approvals/{record_id}/approve")
|
||||
def api_approve(record_id: str, body: ReviewBody, user: Annotated[User, Depends(require_permission("write:approval_review"))]) -> dict[str, Any]:
|
||||
try:
|
||||
return approve_and_execute(
|
||||
record_id,
|
||||
reviewed_by=user.name,
|
||||
reviewed_by_user_id=user.id,
|
||||
comment=body.comment,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@app.post("/api/v1/approvals/{record_id}/reject")
|
||||
def api_reject(record_id: str, body: ReviewBody, user: Annotated[User, Depends(require_permission("write:approval_review"))]) -> dict[str, Any]:
|
||||
try:
|
||||
return reject_approval(
|
||||
record_id,
|
||||
reviewed_by=user.name,
|
||||
reviewed_by_user_id=user.id,
|
||||
comment=body.comment,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@app.post("/api/v1/register-batch")
|
||||
def api_register_batch(body: RegisterBatchBody, user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
|
||||
if body.skip_audit:
|
||||
if not can_submit_action(user, "register_batch"):
|
||||
raise HTTPException(403, "无权登记批次")
|
||||
try:
|
||||
return register_batch(None, body.project, body.task, body.batch, pack=body.pack, stage=body.stage, engineer=body.engineer, location=body.location)
|
||||
except (ValueError, FileNotFoundError) as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
if not can_submit_action(user, "register_batch"):
|
||||
raise HTTPException(403, "无权提交登记审核")
|
||||
return submit_approval(
|
||||
"register_batch",
|
||||
body.model_dump(exclude={"submitted_by", "skip_audit"}),
|
||||
submitted_by=user.name,
|
||||
submitted_by_user_id=user.id,
|
||||
note="登记 batch.meta",
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/v1/data/organize")
|
||||
def api_organize(body: OrganizeBody, _user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
|
||||
try:
|
||||
return organize_batch(Path(body.batch_path), task=body.task)
|
||||
except FileNotFoundError as e:
|
||||
raise HTTPException(404, str(e)) from e
|
||||
|
||||
|
||||
@app.post("/api/v1/data/inspect-upload")
|
||||
def api_inspect_upload(body: InspectUploadBody, _user: Annotated[User, Depends(require_permission("read:catalog"))]) -> dict[str, Any]:
|
||||
try:
|
||||
result = inspect_uploaded_dataset(body.project, body.task, body.source_path)
|
||||
return {"ok": True, "normalized": result.to_dict()}
|
||||
except FileNotFoundError as e:
|
||||
raise HTTPException(404, str(e)) from e
|
||||
except UnknownFormatError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@app.post("/api/v1/data/upload/file")
|
||||
async def api_upload_file(
|
||||
project: Annotated[str, Form()],
|
||||
user: Annotated[User, Depends(require_any_permission("write:approval_submit", "write:approval_submit:register"))],
|
||||
task: Annotated[str | None, Form()] = None,
|
||||
file: UploadFile = File(...),
|
||||
) -> dict[str, Any]:
|
||||
if not file.filename:
|
||||
raise HTTPException(400, "上传文件名不能为空")
|
||||
try:
|
||||
candidate = create_uploaded_candidate(
|
||||
project=project,
|
||||
task=task,
|
||||
original_name=file.filename,
|
||||
upload_size_bytes=0,
|
||||
submitted_by_name=user.name if user else None,
|
||||
submitted_by_user_id=user.id if user else None,
|
||||
)
|
||||
write_candidate_upload(candidate["id"], file.file)
|
||||
job = enqueue_job("analyze_uploaded_dataset", {"candidate_id": candidate["id"]}, async_run=True)
|
||||
link_candidate_analysis_job(candidate["id"], job["id"])
|
||||
updated = get_candidate(candidate["id"]) or candidate
|
||||
return {"ok": True, "candidate": updated, "job": job}
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
finally:
|
||||
await file.close()
|
||||
|
||||
|
||||
@app.get("/api/v1/data/candidates")
|
||||
def api_data_candidates(
|
||||
_user: Annotated[User, Depends(require_permission("read:catalog"))],
|
||||
limit: int = Query(50, ge=1, le=500),
|
||||
) -> dict[str, Any]:
|
||||
return {"items": list_data_candidates(limit=limit)}
|
||||
|
||||
|
||||
@app.get("/api/v1/data/candidates/{candidate_id}")
|
||||
def api_data_candidate(candidate_id: str, _user: Annotated[User, Depends(require_permission("read:catalog"))]) -> dict[str, Any]:
|
||||
item = get_candidate(candidate_id)
|
||||
if not item:
|
||||
raise HTTPException(404, "candidate 不存在")
|
||||
return item
|
||||
|
||||
|
||||
def _mount_ui() -> None:
|
||||
global _UI_DIR
|
||||
for ui in (PLATFORM_WEB, PLATFORM_DIR / "web" / "dist"):
|
||||
if (ui / "index.html").is_file():
|
||||
_UI_DIR = ui
|
||||
app.mount("/assets", StaticFiles(directory=str(ui / "assets")), name="ui-assets")
|
||||
return
|
||||
|
||||
|
||||
_mount_ui()
|
||||
|
||||
|
||||
@app.get("/{full_path:path}", include_in_schema=False)
|
||||
def spa_fallback(full_path: str):
|
||||
if full_path.startswith("api/"):
|
||||
raise HTTPException(404, "Not Found")
|
||||
if not _UI_DIR:
|
||||
raise HTTPException(404, "UI not built")
|
||||
|
||||
safe = Path(full_path).as_posix().lstrip("/")
|
||||
target = (_UI_DIR / safe).resolve()
|
||||
ui_root = _UI_DIR.resolve()
|
||||
if safe and target.is_file() and target.is_relative_to(ui_root):
|
||||
return FileResponse(target)
|
||||
return FileResponse(_UI_DIR / "index.html")
|
||||
|
||||
|
||||
@app.head("/{full_path:path}", include_in_schema=False)
|
||||
def spa_fallback_head(full_path: str):
|
||||
return spa_fallback(full_path)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import uvicorn
|
||||
|
||||
ap = argparse.ArgumentParser(description="华胥智能主动安全平台")
|
||||
ap.add_argument("--host", default="127.0.0.1")
|
||||
ap.add_argument("--port", type=int, default=8787)
|
||||
args = ap.parse_args()
|
||||
uvicorn.run(app, host=args.host, port=args.port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
0
platform/as_platform/audit/__init__.py
Normal file
0
platform/as_platform/audit/__init__.py
Normal file
355
platform/as_platform/audit/preview.py
Normal file
355
platform/as_platform/audit/preview.py
Normal file
@@ -0,0 +1,355 @@
|
||||
"""审核单关联的送标/回传数据:解析范围、列举图像、渲染 GT 叠加。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Sequence
|
||||
|
||||
import yaml
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
from as_platform.data.batch import IMG_EXTS
|
||||
from as_platform.data.core import load_wf, proj_root, resolve_pack_dir
|
||||
|
||||
IMAGE_EXTS = tuple(ext.lower() for ext in IMG_EXTS) + tuple(ext.upper() for ext in IMG_EXTS if ext.islower())
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageRef:
|
||||
image_path: Path
|
||||
label_path: Path | None
|
||||
batch: str
|
||||
location: str
|
||||
split: str
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
key = f"{self.image_path}|{self.label_path or ''}"
|
||||
return hashlib.sha256(key.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def _find_image(images_dir: Path, stem: str) -> Path | None:
|
||||
for ext in IMAGE_EXTS:
|
||||
p = images_dir / f"{stem}{ext}"
|
||||
if p.is_file():
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
def _parse_yolo_line(line: str) -> dict[str, Any] | None:
|
||||
parts = line.strip().split()
|
||||
if len(parts) < 5:
|
||||
return None
|
||||
try:
|
||||
class_id = int(float(parts[0]))
|
||||
cx, cy, w, h = map(float, parts[1:5])
|
||||
except Exception:
|
||||
return None
|
||||
keypoints: list[tuple[float, float, float]] = []
|
||||
rest = parts[5:]
|
||||
if len(rest) >= 3:
|
||||
n = len(rest) // 3
|
||||
for i in range(n):
|
||||
keypoints.append((float(rest[i * 3]), float(rest[i * 3 + 1]), float(rest[i * 3 + 2])))
|
||||
return {"class_id": class_id, "bbox": (cx, cy, w, h), "keypoints": keypoints}
|
||||
|
||||
|
||||
def parse_label_file(label_path: Path) -> list[dict[str, Any]]:
|
||||
if not label_path.is_file():
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
for raw in label_path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
parsed = _parse_yolo_line(raw)
|
||||
if parsed is not None:
|
||||
out.append(parsed)
|
||||
return out
|
||||
|
||||
|
||||
def _load_font(size: int) -> ImageFont.ImageFont:
|
||||
for p in (
|
||||
"/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
|
||||
"/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc",
|
||||
"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
|
||||
):
|
||||
if Path(p).exists():
|
||||
try:
|
||||
return ImageFont.truetype(p, size)
|
||||
except Exception:
|
||||
continue
|
||||
return ImageFont.load_default()
|
||||
|
||||
|
||||
def _yolo_bbox_to_xyxy(bbox: tuple[float, float, float, float], width: int, height: int) -> tuple[int, int, int, int]:
|
||||
cx, cy, bw, bh = bbox
|
||||
x1 = int((cx - bw / 2.0) * width)
|
||||
y1 = int((cy - bh / 2.0) * height)
|
||||
x2 = int((cx + bw / 2.0) * width)
|
||||
y2 = int((cy + bh / 2.0) * height)
|
||||
return (
|
||||
max(0, min(width - 1, x1)),
|
||||
max(0, min(height - 1, y1)),
|
||||
max(0, min(width - 1, x2)),
|
||||
max(0, min(height - 1, y2)),
|
||||
)
|
||||
|
||||
|
||||
def render_overlay(
|
||||
image_path: Path,
|
||||
label_path: Path | None,
|
||||
class_names: dict[int, str],
|
||||
*,
|
||||
max_size: int | None = None,
|
||||
) -> bytes:
|
||||
with Image.open(image_path) as im:
|
||||
base = im.convert("RGB")
|
||||
if max_size and max(base.size) > max_size:
|
||||
base.thumbnail((max_size, max_size), Image.Resampling.LANCZOS)
|
||||
|
||||
over = base.copy()
|
||||
draw = ImageDraw.Draw(over)
|
||||
w, h = over.size
|
||||
font = _load_font(max(12, min(18, w // 40)))
|
||||
anns = parse_label_file(label_path) if label_path else []
|
||||
|
||||
palette = [
|
||||
(220, 20, 60),
|
||||
(30, 144, 255),
|
||||
(50, 205, 50),
|
||||
(255, 165, 0),
|
||||
(186, 85, 211),
|
||||
(0, 206, 209),
|
||||
]
|
||||
for ann in anns:
|
||||
cid = ann["class_id"]
|
||||
color = palette[cid % len(palette)]
|
||||
x1, y1, x2, y2 = _yolo_bbox_to_xyxy(ann["bbox"], w, h)
|
||||
draw.rectangle((x1, y1, x2, y2), outline=color, width=max(2, w // 320))
|
||||
label = class_names.get(cid, f"class_{cid}")
|
||||
draw.text((x1 + 2, max(0, y1 - 16)), label, fill=color, font=font)
|
||||
for kx, ky, kv in ann.get("keypoints") or []:
|
||||
if kv <= 0:
|
||||
continue
|
||||
px, py = int(kx * w), int(ky * h)
|
||||
r = max(2, w // 400)
|
||||
draw.ellipse((px - r, py - r, px + r, py + r), outline=color, fill=color)
|
||||
|
||||
buf = io.BytesIO()
|
||||
over.save(buf, format="JPEG", quality=88)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _collect_from_split(batch_dir: Path, split: str, *, batch: str, location: str) -> list[ImageRef]:
|
||||
if split:
|
||||
images_dir = batch_dir / "images" / split
|
||||
labels_dir = batch_dir / "labels" / split
|
||||
else:
|
||||
images_dir = batch_dir / "images"
|
||||
labels_dir = batch_dir / "labels"
|
||||
|
||||
out: list[ImageRef] = []
|
||||
if not images_dir.is_dir():
|
||||
return out
|
||||
|
||||
label_stems: dict[str, Path] = {}
|
||||
if labels_dir.is_dir():
|
||||
for lp in labels_dir.glob("*.txt"):
|
||||
label_stems[lp.stem] = lp
|
||||
|
||||
seen: set[str] = set()
|
||||
for p in sorted(images_dir.iterdir()):
|
||||
if not p.is_file() or p.suffix.lower() not in {e.lower() for e in IMG_EXTS}:
|
||||
continue
|
||||
stem = p.stem
|
||||
seen.add(stem)
|
||||
out.append(
|
||||
ImageRef(
|
||||
image_path=p.resolve(),
|
||||
label_path=label_stems.get(stem),
|
||||
batch=batch,
|
||||
location=location,
|
||||
split=split or "root",
|
||||
)
|
||||
)
|
||||
|
||||
for stem, lp in sorted(label_stems.items()):
|
||||
if stem in seen:
|
||||
continue
|
||||
img = _find_image(images_dir, stem)
|
||||
if img:
|
||||
out.append(
|
||||
ImageRef(
|
||||
image_path=img.resolve(),
|
||||
label_path=lp.resolve(),
|
||||
batch=batch,
|
||||
location=location,
|
||||
split=split or "root",
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def collect_batch_images(batch_dir: Path, *, batch: str, location: str) -> list[ImageRef]:
|
||||
if not batch_dir.is_dir():
|
||||
return []
|
||||
refs: list[ImageRef] = []
|
||||
for split in ("train", "val", "test", ""):
|
||||
refs.extend(_collect_from_split(batch_dir, split, batch=batch, location=location))
|
||||
# 去重(flat 与 train 可能重叠)
|
||||
dedup: dict[str, ImageRef] = {}
|
||||
for ref in refs:
|
||||
dedup[str(ref.image_path)] = ref
|
||||
return sorted(dedup.values(), key=lambda r: (r.batch, r.split, r.image_path.name))
|
||||
|
||||
|
||||
def _dms_task_cfg(root: Path, wf: dict, task: str) -> tuple[dict, dict]:
|
||||
reg_path = root / wf["projects"]["dms"]["registry"]
|
||||
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
|
||||
if task not in reg.get("tasks", {}):
|
||||
raise ValueError(f"未知 task: {task}")
|
||||
return reg, reg["tasks"][task]
|
||||
|
||||
|
||||
def resolve_approval_scope(action: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
"""解析审核单对应的数据目录与类别名。"""
|
||||
p = params or {}
|
||||
wf = load_wf()
|
||||
|
||||
if action in ("build_dms", "register_batch"):
|
||||
task = p.get("task")
|
||||
if not task:
|
||||
raise ValueError("缺少 task 参数")
|
||||
root = proj_root(wf, "dms")
|
||||
reg, tcfg = _dms_task_cfg(root, wf, task)
|
||||
src_sub = (reg.get("ingest") or {}).get("sources_subdir", "sources")
|
||||
pack = p.get("pack") or "dms_v2"
|
||||
batches: list[dict[str, Any]] = []
|
||||
|
||||
location = p.get("location", "inbox")
|
||||
if action == "build_dms" and p.get("all_sources"):
|
||||
location = "sources"
|
||||
if action == "build_dms" and p.get("batch") and not p.get("all_sources"):
|
||||
location = "inbox"
|
||||
|
||||
if location == "inbox":
|
||||
batch_name = p.get("batch")
|
||||
if batch_name:
|
||||
batches.append({"path": root / "inbox" / task / batch_name, "batch": batch_name, "location": "inbox"})
|
||||
else:
|
||||
ib = root / "inbox" / task
|
||||
if ib.is_dir():
|
||||
for d in sorted(ib.iterdir()):
|
||||
if d.is_dir() and not d.name.startswith("."):
|
||||
batches.append({"path": d, "batch": d.name, "location": "inbox"})
|
||||
else:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack)
|
||||
src_root = pack_dir / tcfg.get("task_dir", task) / src_sub
|
||||
batch_name = p.get("batch")
|
||||
if batch_name:
|
||||
batches.append({"path": src_root / batch_name, "batch": batch_name, "location": "sources"})
|
||||
elif src_root.is_dir():
|
||||
for d in sorted(src_root.iterdir()):
|
||||
if d.is_dir() and d.name not in ("_ingested", "_merged") and not d.name.startswith("."):
|
||||
batches.append({"path": d, "batch": d.name, "location": "sources"})
|
||||
|
||||
names = tcfg.get("names") or {}
|
||||
class_names = {int(k): v for k, v in names.items()} if isinstance(names, dict) else {i: n for i, n in enumerate(names)}
|
||||
return {
|
||||
"project": "dms",
|
||||
"task": task,
|
||||
"pack": pack,
|
||||
"scope_label": f"DMS · {task} · {pack}" + (" · 全部 sources" if location == "sources" and not p.get("batch") else ""),
|
||||
"class_names": class_names,
|
||||
"batches": batches,
|
||||
}
|
||||
|
||||
if action in ("train_dms", "promote_dms", "eval_dms"):
|
||||
task = p.get("task")
|
||||
if not task:
|
||||
raise ValueError("缺少 task 参数")
|
||||
root = proj_root(wf, "dms")
|
||||
_, tcfg = _dms_task_cfg(root, wf, task)
|
||||
pack = p.get("pack") or "dms_v2"
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack)
|
||||
task_dir = pack_dir / tcfg.get("task_dir", task)
|
||||
batches = [{"path": task_dir, "batch": f"{pack}/{task}", "location": "pack"}]
|
||||
names = tcfg.get("names") or {}
|
||||
class_names = {int(k): v for k, v in names.items()} if isinstance(names, dict) else {i: n for i, n in enumerate(names)}
|
||||
label = "模型晋级" if action == "promote_dms" else ("评估" if action == "eval_dms" else "训练")
|
||||
return {
|
||||
"project": "dms",
|
||||
"task": task,
|
||||
"pack": pack,
|
||||
"scope_label": f"DMS · {task} · {pack} · pack 数据({label})",
|
||||
"class_names": class_names,
|
||||
"batches": batches,
|
||||
}
|
||||
|
||||
raise ValueError(f"暂不支持预览的动作: {action}")
|
||||
|
||||
|
||||
def list_scope_images(scope: dict[str, Any], *, offset: int = 0, limit: int = 60) -> dict[str, Any]:
|
||||
all_refs: list[ImageRef] = []
|
||||
for b in scope.get("batches") or []:
|
||||
batch_dir = Path(b["path"])
|
||||
all_refs.extend(
|
||||
collect_batch_images(batch_dir, batch=b.get("batch", batch_dir.name), location=b.get("location", ""))
|
||||
)
|
||||
|
||||
dedup: dict[str, ImageRef] = {str(r.image_path): r for r in all_refs}
|
||||
ordered = sorted(dedup.values(), key=lambda r: (r.batch, r.split, r.image_path.name))
|
||||
total = len(ordered)
|
||||
page = ordered[offset : offset + limit]
|
||||
|
||||
items = []
|
||||
for ref in page:
|
||||
anns = parse_label_file(ref.label_path) if ref.label_path else []
|
||||
items.append(
|
||||
{
|
||||
"id": ref.id,
|
||||
"batch": ref.batch,
|
||||
"location": ref.location,
|
||||
"split": ref.split,
|
||||
"filename": ref.image_path.name,
|
||||
"has_label": ref.label_path is not None and ref.label_path.is_file(),
|
||||
"box_count": len(anns),
|
||||
"missing_label": ref.label_path is None or not ref.label_path.is_file(),
|
||||
}
|
||||
)
|
||||
return {"total": total, "offset": offset, "limit": limit, "items": items}
|
||||
|
||||
|
||||
def find_image_ref(scope: dict[str, Any], image_id: str) -> ImageRef | None:
|
||||
"""线性查找;审核场景批次有限,可接受。"""
|
||||
batches = scope.get("batches") or []
|
||||
for b in batches:
|
||||
batch_dir = Path(b["path"])
|
||||
refs = collect_batch_images(batch_dir, batch=b.get("batch", batch_dir.name), location=b.get("location", ""))
|
||||
for ref in refs:
|
||||
if ref.id == image_id:
|
||||
return ref
|
||||
return None
|
||||
|
||||
|
||||
def image_to_item(ref: ImageRef) -> dict[str, Any]:
|
||||
anns = parse_label_file(ref.label_path) if ref.label_path else []
|
||||
return {
|
||||
"id": ref.id,
|
||||
"batch": ref.batch,
|
||||
"location": ref.location,
|
||||
"split": ref.split,
|
||||
"filename": ref.image_path.name,
|
||||
"has_label": ref.label_path is not None and ref.label_path.is_file(),
|
||||
"box_count": len(anns),
|
||||
"missing_label": ref.label_path is None or not ref.label_path.is_file(),
|
||||
"annotations": [
|
||||
{
|
||||
"class_id": a["class_id"],
|
||||
"class_name": None,
|
||||
"bbox": a["bbox"],
|
||||
"keypoints": a.get("keypoints") or [],
|
||||
}
|
||||
for a in anns
|
||||
],
|
||||
}
|
||||
162
platform/as_platform/audit/queue.py
Normal file
162
platform/as_platform/audit/queue.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""审核队列(SQLite)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import Approval, User
|
||||
from as_platform.config import LANE_DATA_VIZ_ENABLED
|
||||
|
||||
ACTIONS_REQUIRING_APPROVAL = {
|
||||
"build_dms", "build_lane", "enable_pack", "disable_pack",
|
||||
"train_dms", "train_lane", "eval_dms", "promote_dms",
|
||||
"pipeline_dms", "register_batch", "eval_lane", "visualize_dms", "visualize_lane",
|
||||
}
|
||||
|
||||
ACTION_LABELS = {
|
||||
"build_dms": "DMS 入库 (build)",
|
||||
"build_lane": "车道线合并列表 (build lane)",
|
||||
"enable_pack": "启用训练数据包",
|
||||
"disable_pack": "停用训练数据包",
|
||||
"train_dms": "DMS 训练",
|
||||
"train_lane": "车道线训练",
|
||||
"eval_dms": "DMS 评估",
|
||||
"eval_lane": "车道线评估",
|
||||
"visualize_dms": "DMS 检测可视化",
|
||||
"visualize_lane": "车道线可视化",
|
||||
"promote_dms": "DMS 模型晋级",
|
||||
"pipeline_dms": "DMS 半自动流水线",
|
||||
"register_batch": "登记批次元数据",
|
||||
}
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _new_id() -> str:
|
||||
return f"apr-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def submit_approval(
|
||||
action: str,
|
||||
params: dict[str, Any],
|
||||
*,
|
||||
submitted_by: str | None = None,
|
||||
submitted_by_user_id: int | None = None,
|
||||
note: str | None = None,
|
||||
auto_execute: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
if action not in ACTIONS_REQUIRING_APPROVAL:
|
||||
raise ValueError(f"未知动作: {action},允许: {sorted(ACTIONS_REQUIRING_APPROVAL)}")
|
||||
if action == "visualize_lane" and not LANE_DATA_VIZ_ENABLED:
|
||||
raise ValueError("车道线数据可视化暂未开放")
|
||||
|
||||
from as_platform.agents.trace import trace_span
|
||||
|
||||
with session_scope() as db:
|
||||
rec = Approval(
|
||||
id=_new_id(),
|
||||
status="pending",
|
||||
action=action,
|
||||
action_label=ACTION_LABELS.get(action, action),
|
||||
note=note,
|
||||
submitted_by_name=submitted_by,
|
||||
submitted_by_user_id=submitted_by_user_id,
|
||||
submitted_at=_now(),
|
||||
)
|
||||
rec.set_params(params)
|
||||
db.add(rec)
|
||||
db.flush()
|
||||
out = rec.to_dict()
|
||||
|
||||
with trace_span("approval_submit", approval_id=out["id"], action=action):
|
||||
pass
|
||||
|
||||
if auto_execute:
|
||||
return approve_and_execute(out["id"], reviewed_by="system", comment="auto_execute")
|
||||
return out
|
||||
|
||||
|
||||
def list_approvals(status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
|
||||
with session_scope() as db:
|
||||
q = db.query(Approval).order_by(Approval.submitted_at.desc())
|
||||
if status:
|
||||
q = q.filter(Approval.status == status)
|
||||
return [a.to_dict() for a in q.limit(limit).all()]
|
||||
|
||||
|
||||
def get_approval(record_id: str) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
rec = db.get(Approval, record_id)
|
||||
return rec.to_dict() if rec else None
|
||||
|
||||
|
||||
def _update(record_id: str, **patch: Any) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
rec = db.get(Approval, record_id)
|
||||
if not rec:
|
||||
return None
|
||||
for k, v in patch.items():
|
||||
if k == "result" and isinstance(v, dict):
|
||||
rec.set_result(v)
|
||||
elif hasattr(rec, k):
|
||||
setattr(rec, k, v)
|
||||
db.flush()
|
||||
return rec.to_dict()
|
||||
|
||||
|
||||
def approve_and_execute(
|
||||
record_id: str,
|
||||
*,
|
||||
reviewed_by: str | None = None,
|
||||
reviewed_by_user_id: int | None = None,
|
||||
comment: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
rec = get_approval(record_id)
|
||||
if not rec:
|
||||
raise ValueError(f"审核单不存在: {record_id}")
|
||||
if rec.get("status") != "pending":
|
||||
raise ValueError(f"当前状态不可审批: {rec.get('status')}")
|
||||
|
||||
from as_platform.agents.trace import trace_span
|
||||
from as_platform.jobs.queue import enqueue_job
|
||||
|
||||
_update(
|
||||
record_id,
|
||||
status="approved",
|
||||
reviewed_by_name=reviewed_by,
|
||||
reviewed_by_user_id=reviewed_by_user_id,
|
||||
reviewed_at=_now(),
|
||||
review_comment=comment,
|
||||
)
|
||||
|
||||
with trace_span("approval_approved", approval_id=record_id, action=rec["action"]):
|
||||
job = enqueue_job(rec["action"], rec.get("params") or {}, approval_id=record_id, async_run=True)
|
||||
|
||||
_update(record_id, job_id=job.get("id"), status="running")
|
||||
return get_approval(record_id) or {}
|
||||
|
||||
|
||||
def reject_approval(
|
||||
record_id: str,
|
||||
*,
|
||||
reviewed_by: str | None = None,
|
||||
reviewed_by_user_id: int | None = None,
|
||||
comment: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
rec = get_approval(record_id)
|
||||
if not rec:
|
||||
raise ValueError(f"审核单不存在: {record_id}")
|
||||
if rec.get("status") != "pending":
|
||||
raise ValueError(f"当前状态不可驳回: {rec.get('status')}")
|
||||
return _update(
|
||||
record_id,
|
||||
status="rejected",
|
||||
reviewed_by_name=reviewed_by,
|
||||
reviewed_by_user_id=reviewed_by_user_id,
|
||||
reviewed_at=_now(),
|
||||
review_comment=comment,
|
||||
) or {}
|
||||
0
platform/as_platform/auth/__init__.py
Normal file
0
platform/as_platform/auth/__init__.py
Normal file
71
platform/as_platform/auth/deps.py
Normal file
71
platform/as_platform/auth/deps.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""FastAPI 认证依赖。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from sqlalchemy.orm import Session, joinedload
|
||||
|
||||
from as_platform.auth.jwt import decode_access_token
|
||||
from as_platform.config import DEV_AUTH_ENABLED
|
||||
from as_platform.db.engine import get_db
|
||||
from as_platform.db.init_db import user_has_permission
|
||||
from as_platform.db.models import Role, User
|
||||
|
||||
_bearer = HTTPBearer(auto_error=False)
|
||||
|
||||
|
||||
def _load_user(db: Session, user_id: int) -> User | None:
|
||||
return (
|
||||
db.query(User)
|
||||
.options(joinedload(User.roles).joinedload(Role.permissions))
|
||||
.filter(User.id == user_id, User.is_active.is_(True))
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def get_current_user_optional(
|
||||
creds: Annotated[HTTPAuthorizationCredentials | None, Depends(_bearer)],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> User | None:
|
||||
if not creds:
|
||||
return None
|
||||
payload = decode_access_token(creds.credentials)
|
||||
if not payload or "sub" not in payload:
|
||||
return None
|
||||
return _load_user(db, int(payload["sub"]))
|
||||
|
||||
|
||||
def get_current_user(
|
||||
user: Annotated[User | None, Depends(get_current_user_optional)],
|
||||
) -> User:
|
||||
if not user:
|
||||
raise HTTPException(status.HTTP_401_UNAUTHORIZED, detail="未登录,请先飞书登录")
|
||||
return user
|
||||
|
||||
|
||||
def require_permission(permission: str):
|
||||
def _dep(user: Annotated[User, Depends(get_current_user)]) -> User:
|
||||
if not user_has_permission(user, permission):
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, detail=f"缺少权限: {permission}")
|
||||
return user
|
||||
|
||||
return _dep
|
||||
|
||||
|
||||
def require_any_permission(*permissions: str):
|
||||
def _dep(user: Annotated[User, Depends(get_current_user)]) -> User:
|
||||
if any(user_has_permission(user, p) for p in permissions):
|
||||
return user
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="权限不足")
|
||||
|
||||
return _dep
|
||||
|
||||
|
||||
def can_submit_action(user: User, action: str) -> bool:
|
||||
if user_has_permission(user, "write:approval_submit"):
|
||||
return True
|
||||
if action == "register_batch" and user_has_permission(user, "write:approval_submit:register"):
|
||||
return True
|
||||
return False
|
||||
142
platform/as_platform/auth/feishu.py
Normal file
142
platform/as_platform/auth/feishu.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""飞书 OAuth 登录。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from jose import JWTError, jwt
|
||||
|
||||
from as_platform.config import FEISHU_APP_ID, FEISHU_APP_SECRET, FEISHU_REDIRECT_URI, JWT_SECRET
|
||||
|
||||
FEISHU_AUTHORIZE_URL = "https://passport.feishu.cn/suite/passport/oauth/authorize"
|
||||
FEISHU_TOKEN_URL = "https://open.feishu.cn/open-apis/authen/v1/access_token"
|
||||
FEISHU_USER_URL = "https://open.feishu.cn/open-apis/authen/v1/user_info"
|
||||
FEISHU_TENANT_TOKEN_URL = "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal"
|
||||
FEISHU_CONTACT_USER_URL = "https://open.feishu.cn/open-apis/contact/v3/users/{user_id}"
|
||||
|
||||
STATE_ALG = "HS256"
|
||||
STATE_EXPIRE_MINUTES = 10
|
||||
|
||||
|
||||
def is_feishu_configured() -> bool:
|
||||
return bool(FEISHU_APP_ID and FEISHU_APP_SECRET)
|
||||
|
||||
|
||||
def build_authorize_url() -> tuple[str, str]:
|
||||
# 使用签名 state,避免服务重启/多进程导致内存 state 丢失
|
||||
now = datetime.now(timezone.utc)
|
||||
state = jwt.encode(
|
||||
{
|
||||
"nonce": secrets.token_urlsafe(12),
|
||||
"iat": int(now.timestamp()),
|
||||
"exp": int((now + timedelta(minutes=STATE_EXPIRE_MINUTES)).timestamp()),
|
||||
"typ": "feishu_oauth_state",
|
||||
},
|
||||
JWT_SECRET,
|
||||
algorithm=STATE_ALG,
|
||||
)
|
||||
params = {
|
||||
"client_id": FEISHU_APP_ID,
|
||||
"redirect_uri": FEISHU_REDIRECT_URI,
|
||||
"response_type": "code",
|
||||
"state": state,
|
||||
}
|
||||
return f"{FEISHU_AUTHORIZE_URL}?{urllib.parse.urlencode(params)}", state
|
||||
|
||||
|
||||
def verify_state(state: str) -> bool:
|
||||
try:
|
||||
payload = jwt.decode(state, JWT_SECRET, algorithms=[STATE_ALG])
|
||||
return payload.get("typ") == "feishu_oauth_state"
|
||||
except JWTError:
|
||||
return False
|
||||
|
||||
|
||||
def exchange_code(code: str) -> dict[str, Any]:
|
||||
"""用授权码换 user_access_token 并拉取用户信息。"""
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
token_resp = client.post(
|
||||
FEISHU_TOKEN_URL,
|
||||
json={
|
||||
"grant_type": "authorization_code",
|
||||
"code": code,
|
||||
"app_id": FEISHU_APP_ID,
|
||||
"app_secret": FEISHU_APP_SECRET,
|
||||
},
|
||||
)
|
||||
token_resp.raise_for_status()
|
||||
token_data = token_resp.json()
|
||||
if token_data.get("code") != 0:
|
||||
raise RuntimeError(token_data.get("msg") or "飞书 token 交换失败")
|
||||
|
||||
access_token = token_data["data"]["access_token"]
|
||||
user_resp = client.get(
|
||||
FEISHU_USER_URL,
|
||||
headers={"Authorization": f"Bearer {access_token}"},
|
||||
)
|
||||
user_resp.raise_for_status()
|
||||
user_data = user_resp.json()
|
||||
if user_data.get("code") != 0:
|
||||
raise RuntimeError(user_data.get("msg") or "飞书用户信息获取失败")
|
||||
|
||||
info = user_data["data"]
|
||||
department_ids: list[str] = []
|
||||
user_id = info.get("user_id")
|
||||
tenant_key = info.get("tenant_key")
|
||||
open_id = info.get("open_id") or info.get("openId")
|
||||
if open_id:
|
||||
try:
|
||||
tenant_access_token = _get_tenant_access_token(client)
|
||||
contact_user = _get_contact_user_profile(client, tenant_access_token, open_id)
|
||||
user_id = contact_user.get("user_id") or user_id
|
||||
tenant_key = contact_user.get("tenant_key") or tenant_key
|
||||
raw_department_ids = contact_user.get("department_ids")
|
||||
if isinstance(raw_department_ids, list):
|
||||
department_ids = [str(x) for x in raw_department_ids if x]
|
||||
except Exception:
|
||||
# 联系人接口失败时不阻断登录
|
||||
department_ids = []
|
||||
|
||||
return {
|
||||
"open_id": open_id,
|
||||
"union_id": info.get("union_id") or info.get("unionId"),
|
||||
"user_id": user_id,
|
||||
"tenant_key": tenant_key,
|
||||
"department_ids": department_ids,
|
||||
"name": info.get("name") or info.get("en_name") or "飞书用户",
|
||||
"email": info.get("email") or info.get("enterprise_email"),
|
||||
"avatar_url": info.get("avatar_url") or info.get("avatar_big"),
|
||||
}
|
||||
|
||||
|
||||
def _get_tenant_access_token(client: httpx.Client) -> str:
|
||||
resp = client.post(
|
||||
FEISHU_TENANT_TOKEN_URL,
|
||||
json={"app_id": FEISHU_APP_ID, "app_secret": FEISHU_APP_SECRET},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "飞书 tenant token 获取失败")
|
||||
token = data.get("tenant_access_token")
|
||||
if not token:
|
||||
raise RuntimeError("飞书 tenant token 为空")
|
||||
return token
|
||||
|
||||
|
||||
def _get_contact_user_profile(
|
||||
client: httpx.Client, tenant_access_token: str, open_id: str
|
||||
) -> dict[str, Any]:
|
||||
resp = client.get(
|
||||
FEISHU_CONTACT_USER_URL.format(user_id=urllib.parse.quote(open_id, safe="")),
|
||||
params={"user_id_type": "open_id", "department_id_type": "open_department_id"},
|
||||
headers={"Authorization": f"Bearer {tenant_access_token}"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "飞书联系人信息获取失败")
|
||||
return data.get("data", {}).get("user", {})
|
||||
29
platform/as_platform/auth/jwt.py
Normal file
29
platform/as_platform/auth/jwt.py
Normal file
@@ -0,0 +1,29 @@
|
||||
"""JWT 令牌。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from jose import JWTError, jwt
|
||||
|
||||
from as_platform.config import JWT_EXPIRE_HOURS, JWT_SECRET
|
||||
|
||||
ALGORITHM = "HS256"
|
||||
|
||||
|
||||
def create_access_token(user_id: int, extra: dict[str, Any] | None = None) -> str:
|
||||
payload = {
|
||||
"sub": str(user_id),
|
||||
"exp": datetime.now(timezone.utc) + timedelta(hours=JWT_EXPIRE_HOURS),
|
||||
"iat": datetime.now(timezone.utc),
|
||||
}
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
return jwt.encode(payload, JWT_SECRET, algorithm=ALGORITHM)
|
||||
|
||||
|
||||
def decode_access_token(token: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
return jwt.decode(token, JWT_SECRET, algorithms=[ALGORITHM])
|
||||
except JWTError:
|
||||
return None
|
||||
68
platform/as_platform/auth/users.py
Normal file
68
platform/as_platform/auth/users.py
Normal file
@@ -0,0 +1,68 @@
|
||||
"""用户服务。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session, joinedload
|
||||
|
||||
from as_platform.db.init_db import assign_default_role
|
||||
from as_platform.db.models import Role, User
|
||||
|
||||
|
||||
def get_user_by_id(db: Session, user_id: int) -> User | None:
|
||||
return (
|
||||
db.query(User)
|
||||
.options(joinedload(User.roles).joinedload(Role.permissions))
|
||||
.filter(User.id == user_id)
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def upsert_feishu_user(db: Session, info: dict[str, Any]) -> User:
|
||||
open_id = info.get("open_id")
|
||||
user = db.query(User).filter(User.feishu_open_id == open_id).first()
|
||||
if not user:
|
||||
user = User(feishu_open_id=open_id)
|
||||
db.add(user)
|
||||
user.feishu_union_id = info.get("union_id") or user.feishu_union_id
|
||||
user.feishu_user_id = info.get("user_id") or user.feishu_user_id
|
||||
user.feishu_tenant_key = info.get("tenant_key") or user.feishu_tenant_key
|
||||
department_ids = info.get("department_ids")
|
||||
if isinstance(department_ids, list):
|
||||
user.feishu_department_ids_json = json.dumps(department_ids, ensure_ascii=False)
|
||||
user.name = info.get("name") or user.name
|
||||
user.email = info.get("email") or user.email
|
||||
user.avatar_url = info.get("avatar_url") or user.avatar_url
|
||||
user.is_active = True
|
||||
db.flush()
|
||||
assign_default_role(db, user)
|
||||
db.refresh(user)
|
||||
return get_user_by_id(db, user.id) # type: ignore
|
||||
|
||||
|
||||
def get_or_create_dev_user(db: Session, name: str = "开发用户") -> User:
|
||||
user = db.query(User).filter(User.name == name, User.feishu_open_id.is_(None)).first()
|
||||
if not user:
|
||||
user = User(name=name, email="dev@local")
|
||||
db.add(user)
|
||||
db.flush()
|
||||
admin = db.query(Role).filter_by(code="admin").first()
|
||||
if admin:
|
||||
user.roles = [admin]
|
||||
db.flush()
|
||||
return get_user_by_id(db, user.id) # type: ignore
|
||||
|
||||
|
||||
def list_users(db: Session) -> list[User]:
|
||||
return db.query(User).options(joinedload(User.roles)).order_by(User.id).all()
|
||||
|
||||
|
||||
def set_user_roles(db: Session, user_id: int, role_codes: list[str]) -> User | None:
|
||||
user = get_user_by_id(db, user_id)
|
||||
if not user:
|
||||
return None
|
||||
roles = db.query(Role).filter(Role.code.in_(role_codes)).all()
|
||||
user.roles = roles
|
||||
db.flush()
|
||||
return user
|
||||
71
platform/as_platform/config.py
Normal file
71
platform/as_platform/config.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""华胥卡车主动安全(AEB)平台根配置。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
AS_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
WORKSPACE = AS_ROOT
|
||||
|
||||
PLATFORM_DIR = AS_ROOT / "platform"
|
||||
PLATFORM_WEB = PLATFORM_DIR / "web" / "dist"
|
||||
ALGORITHMS_DIR = AS_ROOT / "algorithms"
|
||||
DATASETS_DIR = AS_ROOT / "datasets"
|
||||
ALGORITHMS_REGISTRY = ALGORITHMS_DIR / "registry.yaml"
|
||||
|
||||
MANIFESTS = AS_ROOT / "manifests"
|
||||
JOB_LOG = MANIFESTS / "job_log.jsonl"
|
||||
APPROVAL_QUEUE = MANIFESTS / "approval_queue.jsonl"
|
||||
TRACE_LOG = MANIFESTS / "trace_log.jsonl"
|
||||
GRAPH_STATE_DIR = MANIFESTS / "graph_state"
|
||||
ENGINES_REGISTRY = ALGORITHMS_REGISTRY
|
||||
SQLITE_LEGACY_PATH = MANIFESTS / "platform.db"
|
||||
|
||||
# ── 数据库(默认 PostgreSQL)──
|
||||
DB_HOST = os.environ.get("AS_DB_HOST", "127.0.0.1")
|
||||
DB_PORT = os.environ.get("AS_DB_PORT", "5432")
|
||||
DB_USER = os.environ.get("AS_DB_USER", "as_platform")
|
||||
DB_PASSWORD = os.environ.get("AS_DB_PASSWORD", "as_platform")
|
||||
DB_NAME = os.environ.get("AS_DB_NAME", "as_platform")
|
||||
|
||||
|
||||
def build_database_url() -> str:
|
||||
explicit = os.environ.get("AS_DATABASE_URL", "").strip()
|
||||
if explicit:
|
||||
return explicit
|
||||
pwd = quote_plus(DB_PASSWORD)
|
||||
return f"postgresql+psycopg2://{DB_USER}:{pwd}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
|
||||
|
||||
|
||||
DATABASE_URL = build_database_url()
|
||||
IS_POSTGRES = DATABASE_URL.startswith("postgresql")
|
||||
IS_SQLITE = DATABASE_URL.startswith("sqlite")
|
||||
|
||||
# ── Redis(Job 队列 / 事件,docker compose 默认 redis:6379)──
|
||||
REDIS_URL = os.environ.get("AS_REDIS_URL", "redis://127.0.0.1:6379/0")
|
||||
JOB_QUEUE_KEY = os.environ.get("AS_JOB_QUEUE_KEY", "as:job_queue")
|
||||
# thread: API 进程内执行(本机 run_local.sh)| worker: 推 Redis,由 worker 容器/脚本消费
|
||||
JOB_EXECUTOR = os.environ.get("AS_JOB_EXECUTOR", "thread")
|
||||
|
||||
# ── 认证 / 飞书 ──
|
||||
JWT_SECRET = os.environ.get("AS_JWT_SECRET", "change-me-in-production")
|
||||
JWT_EXPIRE_HOURS = int(os.environ.get("AS_JWT_EXPIRE_HOURS", "168"))
|
||||
FEISHU_APP_ID = os.environ.get("FEISHU_APP_ID", "")
|
||||
FEISHU_APP_SECRET = os.environ.get("FEISHU_APP_SECRET", "")
|
||||
FEISHU_REDIRECT_URI = os.environ.get(
|
||||
"FEISHU_REDIRECT_URI",
|
||||
"http://127.0.0.1:8787/api/v1/auth/feishu/callback",
|
||||
)
|
||||
FRONTEND_URL = os.environ.get("AS_FRONTEND_URL", "http://127.0.0.1:8787")
|
||||
DEV_AUTH_ENABLED = os.environ.get("AS_DEV_AUTH", "").lower() in ("1", "true", "yes")
|
||||
FEISHU_ADMIN_OPEN_IDS = {
|
||||
x.strip() for x in os.environ.get("FEISHU_ADMIN_OPEN_IDS", "").split(",") if x.strip()
|
||||
}
|
||||
FEISHU_ADMIN_DEPARTMENT_IDS = {
|
||||
x.strip() for x in os.environ.get("FEISHU_ADMIN_DEPARTMENT_IDS", "").split(",") if x.strip()
|
||||
}
|
||||
|
||||
# ── 功能开关 ──
|
||||
# 车道线 catalog 质检统计(mask 抽样,较慢);暂时默认关闭
|
||||
LANE_DATA_VIZ_ENABLED = os.environ.get("AS_LANE_DATA_VIZ", "").lower() in ("1", "true", "yes")
|
||||
37
platform/as_platform/data/__init__.py
Normal file
37
platform/as_platform/data/__init__.py
Normal file
@@ -0,0 +1,37 @@
|
||||
from as_platform.data.batch import META_FILENAME
|
||||
from as_platform.data.core import (
|
||||
get_catalog,
|
||||
get_pending_report,
|
||||
load_wf,
|
||||
proj_root,
|
||||
register_batch,
|
||||
resolve_pack,
|
||||
resolve_pack_dir,
|
||||
)
|
||||
from as_platform.data.ingest import inspect_uploaded_dataset
|
||||
from as_platform.data.lake import (
|
||||
analyze_uploaded_candidate,
|
||||
create_uploaded_candidate,
|
||||
get_candidate,
|
||||
link_candidate_analysis_job,
|
||||
list_candidates,
|
||||
write_candidate_upload,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"META_FILENAME",
|
||||
"get_pending_report",
|
||||
"get_catalog",
|
||||
"register_batch",
|
||||
"load_wf",
|
||||
"proj_root",
|
||||
"resolve_pack",
|
||||
"resolve_pack_dir",
|
||||
"inspect_uploaded_dataset",
|
||||
"create_uploaded_candidate",
|
||||
"write_candidate_upload",
|
||||
"list_candidates",
|
||||
"get_candidate",
|
||||
"link_candidate_analysis_job",
|
||||
"analyze_uploaded_candidate",
|
||||
]
|
||||
161
platform/as_platform/data/batch.py
Normal file
161
platform/as_platform/data/batch.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""批次元数据 batch.meta.yaml 与目录结构推断。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
META_FILENAME = "batch.meta.yaml"
|
||||
SCHEMA = "huaxu-batch-v1"
|
||||
IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".JPG", ".JPEG", ".PNG"}
|
||||
|
||||
|
||||
def count_images(dir_path: Path) -> int:
|
||||
if not dir_path.is_dir():
|
||||
return 0
|
||||
n = 0
|
||||
for p in dir_path.rglob("*"):
|
||||
if p.is_file() and p.suffix in IMG_EXTS:
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def count_label_files(dir_path: Path) -> int:
|
||||
if not dir_path.is_dir():
|
||||
return 0
|
||||
n = 0
|
||||
for p in dir_path.rglob("*"):
|
||||
if p.is_file() and p.suffix.lower() in (".txt", ".xml"):
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def dms_has_images(batch_dir: Path) -> bool:
|
||||
for sub in ("images", "images/train"):
|
||||
if count_images(batch_dir / sub) > 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def dms_has_labels(batch_dir: Path) -> bool:
|
||||
for sub in ("labels", "labels/train"):
|
||||
d = batch_dir / sub
|
||||
if d.is_dir() and count_label_files(d) > 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def infer_dms_stage(batch_dir: Path) -> str:
|
||||
has_img = dms_has_images(batch_dir)
|
||||
has_lab = dms_has_labels(batch_dir)
|
||||
if has_img and has_lab:
|
||||
return "returned"
|
||||
if has_img:
|
||||
return "raw_pool"
|
||||
return "raw_pool"
|
||||
|
||||
|
||||
def infer_lane_stage(path: Path) -> str:
|
||||
if (path / "list" / "train_gt.txt").is_file():
|
||||
return "ingested"
|
||||
if (path / "train_val_gt.txt").is_file():
|
||||
return "returned"
|
||||
if any(path.glob("**/train_val_gt.txt")):
|
||||
return "returned"
|
||||
if count_images(path) > 0:
|
||||
return "raw_pool"
|
||||
return "raw_pool"
|
||||
|
||||
|
||||
def read_meta(batch_dir: Path) -> dict[str, Any] | None:
|
||||
p = batch_dir / META_FILENAME
|
||||
if not p.is_file():
|
||||
return None
|
||||
try:
|
||||
data = yaml.safe_load(p.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def write_meta(batch_dir: Path, data: dict[str, Any]) -> Path:
|
||||
batch_dir.mkdir(parents=True, exist_ok=True)
|
||||
data.setdefault("schema", SCHEMA)
|
||||
p = batch_dir / META_FILENAME
|
||||
p.write_text(
|
||||
yaml.dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return p
|
||||
|
||||
|
||||
def enrich_batch(
|
||||
batch_dir: Path,
|
||||
*,
|
||||
project: str,
|
||||
task: str | None = None,
|
||||
pack: str | None = None,
|
||||
batch: str,
|
||||
location: str,
|
||||
) -> dict[str, Any]:
|
||||
meta = read_meta(batch_dir) or {}
|
||||
if project == "dms":
|
||||
stage = meta.get("stage") or infer_dms_stage(batch_dir)
|
||||
img_n = meta.get("counts", {}).get("images")
|
||||
lab_n = meta.get("counts", {}).get("labels")
|
||||
if img_n is None:
|
||||
img_n = count_images(batch_dir / "images") + count_images(batch_dir / "images" / "train")
|
||||
if lab_n is None:
|
||||
lab_n = count_label_files(batch_dir / "labels") + count_label_files(batch_dir / "labels" / "train")
|
||||
fmt = meta.get("format", "yolo")
|
||||
else:
|
||||
stage = meta.get("stage") or infer_lane_stage(batch_dir)
|
||||
img_n = meta.get("counts", {}).get("images") or count_images(batch_dir)
|
||||
lab_n = meta.get("counts", {}).get("labels")
|
||||
fmt = meta.get("format", "ufld_archive")
|
||||
if (batch_dir / "list" / "train_gt.txt").is_file():
|
||||
try:
|
||||
lab_n = sum(1 for _ in (batch_dir / "list" / "train_gt.txt").open(encoding="utf-8"))
|
||||
except OSError:
|
||||
lab_n = lab_n or 0
|
||||
|
||||
return {
|
||||
"project": project,
|
||||
"task": task or meta.get("task"),
|
||||
"pack": pack or meta.get("pack"),
|
||||
"batch": batch,
|
||||
"stage": stage,
|
||||
"location": location,
|
||||
"path": str(batch_dir.resolve()),
|
||||
"engineer": meta.get("engineer"),
|
||||
"format": fmt,
|
||||
"counts": {"images": img_n, "labels": lab_n},
|
||||
"has_meta": bool(meta),
|
||||
"next_cli": suggest_cli(project, task, pack, batch, stage, location),
|
||||
}
|
||||
|
||||
|
||||
def suggest_cli(
|
||||
project: str,
|
||||
task: str | None,
|
||||
pack: str | None,
|
||||
batch: str,
|
||||
stage: str,
|
||||
location: str,
|
||||
) -> str:
|
||||
if project == "dms":
|
||||
p = pack or "dms_v2"
|
||||
t = task or "<task>"
|
||||
if location == "inbox" and stage == "returned":
|
||||
return f"python as.py build dms {t} --pack {p} --batch {batch}"
|
||||
if location == "sources" and stage == "returned":
|
||||
return f"python as.py build dms {t} --pack {p} --all-sources"
|
||||
if stage == "raw_pool":
|
||||
return f"# 送标完成后放入 labels,或: python as.py register-batch dms {t} --batch {batch} --stage returned"
|
||||
return f"python as.py build dms {t} --pack {p}"
|
||||
if stage == "returned":
|
||||
return "python as.py add lane --src <archive> --engineer <name> --date YYYYMMDD"
|
||||
if stage == "ingested":
|
||||
return f"python as.py enable lane {pack or '<pack>'} && python as.py build lane"
|
||||
return "python as.py add lane --src <path> --engineer <name> --date YYYYMMDD"
|
||||
246
platform/as_platform/data/catalog_cache.py
Normal file
246
platform/as_platform/data/catalog_cache.py
Normal file
@@ -0,0 +1,246 @@
|
||||
"""Catalog cache: memory/disk + cheap directory-change invalidation."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
|
||||
CATALOG_CACHE_FILE = WORKSPACE / "manifests" / "catalog_cache.json"
|
||||
CATALOG_CACHE_TTL_SEC = int(os.environ.get("AS_CATALOG_CACHE_TTL_SEC", "300"))
|
||||
CATALOG_USE_REPORTS = os.environ.get("AS_CATALOG_USE_REPORTS", "1").lower() in ("1", "true", "yes")
|
||||
CATALOG_CACHE_VERSION = 3
|
||||
|
||||
REPORTS_DIR = WORKSPACE / "reports"
|
||||
DMS_SUMMARY_CSV = REPORTS_DIR / "dms_task_image_summary.csv"
|
||||
DMS_CLASS_CSV = REPORTS_DIR / "dms_task_class_image_counts.csv"
|
||||
|
||||
_CATALOG_MEM_CACHE: dict[str, Any] | None = None
|
||||
|
||||
|
||||
def invalidate_catalog_cache() -> None:
|
||||
global _CATALOG_MEM_CACHE
|
||||
_CATALOG_MEM_CACHE = None
|
||||
if CATALOG_CACHE_FILE.is_file():
|
||||
try:
|
||||
CATALOG_CACHE_FILE.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _dir_fingerprint(path: Path, *, scan_children: bool = True) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"path": str(path), "missing": True}
|
||||
try:
|
||||
st = path.stat()
|
||||
fp: dict[str, Any] = {
|
||||
"path": str(path),
|
||||
"mtime_ns": st.st_mtime_ns,
|
||||
"size": st.st_size,
|
||||
}
|
||||
if scan_children and path.is_dir():
|
||||
children: list[dict[str, Any]] = []
|
||||
try:
|
||||
with os.scandir(path) as it:
|
||||
for entry in it:
|
||||
if entry.name.startswith("."):
|
||||
continue
|
||||
try:
|
||||
est = entry.stat(follow_symlinks=False)
|
||||
children.append(
|
||||
{
|
||||
"name": entry.name,
|
||||
"mtime_ns": est.st_mtime_ns,
|
||||
"is_dir": entry.is_dir(follow_symlinks=False),
|
||||
}
|
||||
)
|
||||
except OSError:
|
||||
children.append({"name": entry.name, "error": True})
|
||||
except OSError:
|
||||
fp["scan_error"] = True
|
||||
children.sort(key=lambda x: x.get("name", ""))
|
||||
fp["children"] = children
|
||||
return fp
|
||||
except OSError:
|
||||
return {"path": str(path), "error": True}
|
||||
|
||||
|
||||
def build_catalog_signature(wf: dict, proj_root_fn) -> dict[str, Any]:
|
||||
"""Cheap signature: config files + inbox/pack directory mtimes (auto-invalidate on drop)."""
|
||||
from as_platform.data.core import _pack_registry_path, load_pack_registry
|
||||
|
||||
files: list[dict[str, Any]] = []
|
||||
for rel in ("workflow.registry.yaml",):
|
||||
p = WORKSPACE / rel
|
||||
try:
|
||||
st = p.stat()
|
||||
files.append({"path": str(p), "mtime_ns": st.st_mtime_ns, "size": st.st_size})
|
||||
except FileNotFoundError:
|
||||
files.append({"path": str(p), "missing": True})
|
||||
|
||||
dirs: list[dict[str, Any]] = []
|
||||
for pname in ("dms", "lane"):
|
||||
root = proj_root_fn(wf, pname)
|
||||
dirs.append(_dir_fingerprint(root, scan_children=False))
|
||||
dirs.append(_dir_fingerprint(root / "inbox"))
|
||||
try:
|
||||
packs_reg = load_pack_registry(pname, root, wf)
|
||||
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
||||
packs_reg = {"packs": []}
|
||||
dirs.append(_dir_fingerprint(root / "packs", scan_children=False))
|
||||
for p in packs_reg.get("packs", []):
|
||||
pack_path = root / p.get("path", p.get("name", ""))
|
||||
dirs.append(_dir_fingerprint(pack_path, scan_children=False))
|
||||
if pname == "dms":
|
||||
for cfg_path in (
|
||||
root / wf["projects"]["dms"]["registry"],
|
||||
root / "manifests" / "dataset_class_summary.txt",
|
||||
_pack_registry_path("dms", root, wf),
|
||||
):
|
||||
try:
|
||||
st = cfg_path.stat()
|
||||
files.append({"path": str(cfg_path), "mtime_ns": st.st_mtime_ns, "size": st.st_size})
|
||||
except FileNotFoundError:
|
||||
files.append({"path": str(cfg_path), "missing": True})
|
||||
reg_path = root / wf["projects"]["dms"]["registry"]
|
||||
if reg_path.is_file():
|
||||
import yaml
|
||||
|
||||
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
|
||||
for task in (reg.get("tasks") or {}).keys():
|
||||
dirs.append(_dir_fingerprint(root / "inbox" / task))
|
||||
if pname == "lane":
|
||||
dirs.append(_dir_fingerprint(_pack_registry_path("lane", root, wf), scan_children=False))
|
||||
|
||||
for csv_path in (DMS_SUMMARY_CSV, DMS_CLASS_CSV):
|
||||
try:
|
||||
st = csv_path.stat()
|
||||
files.append({"path": str(csv_path), "mtime_ns": st.st_mtime_ns, "size": st.st_size})
|
||||
except FileNotFoundError:
|
||||
files.append({"path": str(csv_path), "missing": True})
|
||||
|
||||
return {"files": files, "dirs": dirs}
|
||||
|
||||
|
||||
def load_disk_cache() -> dict[str, Any] | None:
|
||||
if not CATALOG_CACHE_FILE.is_file():
|
||||
return None
|
||||
try:
|
||||
return json.loads(CATALOG_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def save_disk_cache(payload: dict[str, Any]) -> None:
|
||||
CATALOG_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
CATALOG_CACHE_FILE.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
|
||||
def _catalog_has_empty_bbox(catalog: dict[str, Any]) -> bool:
|
||||
for task in (catalog.get("dms") or {}).values():
|
||||
for pack in task.get("packs") or []:
|
||||
boxes = int(pack.get("total_boxes") or 0)
|
||||
pts = pack.get("bbox_points") or []
|
||||
if boxes > 0 and not pts:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_cached_catalog(signature: dict[str, Any], *, refresh: bool = False) -> tuple[dict[str, Any] | None, dict[str, Any]]:
|
||||
"""Return (catalog_data, meta). meta describes cache hit/miss."""
|
||||
global _CATALOG_MEM_CACHE
|
||||
now = time.time()
|
||||
meta: dict[str, Any] = {"cached": False, "source": "scan"}
|
||||
|
||||
if refresh:
|
||||
_CATALOG_MEM_CACHE = None
|
||||
return None, meta
|
||||
|
||||
for source, cache in (("memory", _CATALOG_MEM_CACHE), ("disk", load_disk_cache() if not _CATALOG_MEM_CACHE else None)):
|
||||
if not cache:
|
||||
continue
|
||||
age = now - float(cache.get("generated_at_ts", 0.0))
|
||||
if cache.get("signature") != signature:
|
||||
continue
|
||||
if cache.get("version") != CATALOG_CACHE_VERSION:
|
||||
continue
|
||||
data = cache.get("data") or {}
|
||||
if _catalog_has_empty_bbox(data):
|
||||
continue
|
||||
if age > CATALOG_CACHE_TTL_SEC:
|
||||
continue
|
||||
if source == "disk":
|
||||
_CATALOG_MEM_CACHE = cache
|
||||
meta = {
|
||||
"cached": True,
|
||||
"cache_source": source,
|
||||
"cache_age_sec": round(age, 1),
|
||||
"generated_at_ts": cache.get("generated_at_ts"),
|
||||
"build_source": cache.get("build_source", "scan"),
|
||||
}
|
||||
return cache.get("data", {}), meta
|
||||
|
||||
return None, meta
|
||||
|
||||
|
||||
def store_catalog_cache(signature: dict[str, Any], data: dict[str, Any], *, build_source: str = "scan") -> dict[str, Any]:
|
||||
global _CATALOG_MEM_CACHE
|
||||
now = time.time()
|
||||
payload = {
|
||||
"version": CATALOG_CACHE_VERSION,
|
||||
"generated_at_ts": now,
|
||||
"signature": signature,
|
||||
"build_source": build_source,
|
||||
"data": data,
|
||||
}
|
||||
_CATALOG_MEM_CACHE = payload
|
||||
save_disk_cache(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def load_dms_reports() -> tuple[dict[tuple[str, str], dict[str, int]], dict[str, dict[str, int]]] | None:
|
||||
"""Parse precomputed CSV reports: (task, pack) -> splits, task -> class_counts."""
|
||||
if not CATALOG_USE_REPORTS or not DMS_SUMMARY_CSV.is_file():
|
||||
return None
|
||||
|
||||
splits: dict[tuple[str, str], dict[str, int]] = {}
|
||||
try:
|
||||
with DMS_SUMMARY_CSV.open(encoding="utf-8") as f:
|
||||
for row in csv.DictReader(f):
|
||||
task = row.get("任务", "").strip()
|
||||
pack = row.get("数据包", "").strip() or "default"
|
||||
if not task:
|
||||
continue
|
||||
splits[(task, pack)] = {
|
||||
"train": int(row.get("训练集图片") or 0),
|
||||
"val": int(row.get("验证集图片") or 0),
|
||||
"test": int(row.get("测试集图片") or 0),
|
||||
"total": int(row.get("图片总数") or 0),
|
||||
}
|
||||
except (OSError, ValueError, csv.Error):
|
||||
return None
|
||||
|
||||
class_by_task: dict[str, dict[str, int]] = {}
|
||||
if DMS_CLASS_CSV.is_file():
|
||||
try:
|
||||
with DMS_CLASS_CSV.open(encoding="utf-8") as f:
|
||||
for row in csv.DictReader(f):
|
||||
task = row.get("任务", "").strip()
|
||||
cls_name = row.get("类别名", "").strip()
|
||||
if not task or not cls_name:
|
||||
continue
|
||||
try:
|
||||
cnt = int(row.get("含该类别图片数") or 0)
|
||||
except ValueError:
|
||||
continue
|
||||
class_by_task.setdefault(task, {})[cls_name] = cnt
|
||||
except (OSError, ValueError, csv.Error):
|
||||
pass
|
||||
|
||||
if not splits:
|
||||
return None
|
||||
return splits, class_by_task
|
||||
826
platform/as_platform/data/core.py
Normal file
826
platform/as_platform/data/core.py
Normal file
@@ -0,0 +1,826 @@
|
||||
"""平台共享逻辑:pending、catalog、register-batch。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from as_platform.config import WORKSPACE, LANE_DATA_VIZ_ENABLED
|
||||
from as_platform.data.batch import META_FILENAME, enrich_batch, write_meta
|
||||
from as_platform.data.catalog_cache import (
|
||||
build_catalog_signature,
|
||||
get_cached_catalog,
|
||||
invalidate_catalog_cache,
|
||||
load_dms_reports,
|
||||
store_catalog_cache,
|
||||
)
|
||||
|
||||
MAX_LABEL_FILES_PER_PACK = 2000
|
||||
MAX_BBOX_POINTS_PER_PACK = 1500
|
||||
MAX_LANE_MASK_SAMPLES_PER_PACK = 500
|
||||
LANE_Y_BINS = 12
|
||||
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
|
||||
|
||||
|
||||
def load_wf() -> dict:
|
||||
return yaml.safe_load((WORKSPACE / "workflow.registry.yaml").read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def proj_root(wf: dict, name: str) -> Path:
|
||||
return (WORKSPACE / wf["projects"][name]["root"]).resolve()
|
||||
|
||||
|
||||
def load_pack_registry(project: str, root: Path, wf: dict) -> dict:
|
||||
pcfg = wf["projects"][project]
|
||||
reg_file = root / pcfg.get("packs_registry", "datasets_registry.json")
|
||||
if reg_file.suffix in (".yaml", ".yml"):
|
||||
return yaml.safe_load(reg_file.read_text(encoding="utf-8"))
|
||||
return json.loads(reg_file.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _pack_registry_path(project: str, root: Path, wf: dict) -> Path:
|
||||
pcfg = wf["projects"][project]
|
||||
return root / pcfg.get("packs_registry", "datasets_registry.json")
|
||||
|
||||
|
||||
def resolve_pack(project: str, root: Path, wf: dict, name: str) -> str:
|
||||
reg = load_pack_registry(project, root, wf)
|
||||
name = reg.get("aliases", {}).get(name, name)
|
||||
for p in reg.get("packs", []):
|
||||
if p.get("name") == name:
|
||||
return p.get("path", name)
|
||||
if (root / name).is_dir():
|
||||
return name
|
||||
known = [p.get("name") for p in reg.get("packs", [])]
|
||||
raise ValueError(f"[{project}] 未知包: {name},已登记: {known}")
|
||||
|
||||
|
||||
def resolve_pack_dir(project: str, root: Path, wf: dict, name: str) -> Path:
|
||||
return (root / resolve_pack(project, root, wf, name)).resolve()
|
||||
|
||||
|
||||
def _read_jsonl_tail(path: Path, n: int = 10) -> list[dict]:
|
||||
if not path.is_file():
|
||||
return []
|
||||
lines = path.read_text(encoding="utf-8").strip().splitlines()
|
||||
out = []
|
||||
for line in lines[-n:]:
|
||||
try:
|
||||
out.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def get_pending_report(wf: dict | None = None) -> dict[str, Any]:
|
||||
wf = wf or load_wf()
|
||||
report: dict[str, Any] = {
|
||||
"workspace": str(WORKSPACE),
|
||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"projects": {},
|
||||
"batches": [],
|
||||
}
|
||||
|
||||
for pname, pcfg in wf["projects"].items():
|
||||
root = proj_root(wf, pname)
|
||||
active = set(pcfg.get("active_packs", []))
|
||||
try:
|
||||
reg_all = load_pack_registry(pname, root, wf)
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
reg_all = {"packs": []}
|
||||
all_names = {p["name"] for p in reg_all.get("packs", [])}
|
||||
not_active = sorted(all_names - active)
|
||||
|
||||
proj: dict[str, Any] = {
|
||||
"root": str(root),
|
||||
"active_packs": list(active),
|
||||
"not_enabled": not_active,
|
||||
"tasks": {},
|
||||
"task_defs": {},
|
||||
}
|
||||
|
||||
if pname == "dms":
|
||||
reg_path = root / pcfg["registry"]
|
||||
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
|
||||
src_sub = (reg.get("ingest") or {}).get("sources_subdir", "sources")
|
||||
ingest_log = root / "manifests" / "ingest_log.jsonl"
|
||||
proj["recent_ingest"] = _read_jsonl_tail(ingest_log, 10)
|
||||
|
||||
for task, tcfg in reg.get("tasks", {}).items():
|
||||
proj["task_defs"][task] = {
|
||||
"type": tcfg.get("type"),
|
||||
"nc": tcfg.get("nc"),
|
||||
"names": tcfg.get("names"),
|
||||
"task_dir": tcfg.get("task_dir", task),
|
||||
}
|
||||
inbox_batches: list[str] = []
|
||||
ib = root / "inbox" / task
|
||||
if ib.is_dir():
|
||||
inbox_batches = [
|
||||
d.name for d in ib.iterdir()
|
||||
if d.is_dir() and not d.name.startswith(".")
|
||||
]
|
||||
|
||||
sources_pending: dict[str, list[str]] = {}
|
||||
for pack_name in all_names:
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
src_root = pack_dir / tcfg["task_dir"] / src_sub
|
||||
if src_root.is_dir():
|
||||
batches = [
|
||||
d.name for d in src_root.iterdir()
|
||||
if d.is_dir()
|
||||
and d.name not in ("_ingested", "_merged")
|
||||
and not d.name.startswith(".")
|
||||
]
|
||||
if batches:
|
||||
sources_pending[pack_name] = batches
|
||||
|
||||
proj["tasks"][task] = {
|
||||
"inbox": inbox_batches,
|
||||
"sources": sources_pending,
|
||||
}
|
||||
|
||||
for batch_name in inbox_batches:
|
||||
batch_dir = ib / batch_name
|
||||
report["batches"].append(
|
||||
enrich_batch(
|
||||
batch_dir,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=None,
|
||||
batch=batch_name,
|
||||
location="inbox",
|
||||
)
|
||||
)
|
||||
|
||||
for pack_name, batch_list in sources_pending.items():
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
src_root = pack_dir / tcfg["task_dir"] / src_sub
|
||||
for batch_name in batch_list:
|
||||
batch_dir = src_root / batch_name
|
||||
report["batches"].append(
|
||||
enrich_batch(
|
||||
batch_dir,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=pack_name,
|
||||
batch=batch_name,
|
||||
location="sources",
|
||||
)
|
||||
)
|
||||
|
||||
if pname == "lane":
|
||||
proj["packs"] = {}
|
||||
for pack_name in all_names:
|
||||
try:
|
||||
path = resolve_pack("lane", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
pack_path = root / path
|
||||
train_lines = 0
|
||||
tg = pack_path / "list" / "train_gt.txt"
|
||||
if tg.is_file():
|
||||
train_lines = sum(1 for _ in tg.open(encoding="utf-8"))
|
||||
proj["packs"][pack_name] = {
|
||||
"path": path,
|
||||
"train_lines": train_lines,
|
||||
"enabled": pack_name in active,
|
||||
}
|
||||
if pack_name in not_active and pack_path.is_dir():
|
||||
report["batches"].append(
|
||||
enrich_batch(
|
||||
pack_path,
|
||||
project="lane",
|
||||
task=None,
|
||||
pack=pack_name,
|
||||
batch=path,
|
||||
location="pack",
|
||||
)
|
||||
)
|
||||
|
||||
for child in sorted(root.iterdir()) if root.is_dir() else []:
|
||||
if not child.is_dir() or child.name.startswith("."):
|
||||
continue
|
||||
if child.name in ("lists_merged", "scripts", "inbox"):
|
||||
continue
|
||||
if not child.name.startswith("DATASET-AddBy-"):
|
||||
continue
|
||||
if any(
|
||||
p.get("path") == child.name or p.get("name") == child.name
|
||||
for p in reg_all.get("packs", [])
|
||||
):
|
||||
continue
|
||||
report["batches"].append(
|
||||
enrich_batch(
|
||||
child,
|
||||
project="lane",
|
||||
task=None,
|
||||
pack=None,
|
||||
batch=child.name,
|
||||
location="unregistered",
|
||||
)
|
||||
)
|
||||
|
||||
inbox_lane = root / "inbox"
|
||||
if inbox_lane.is_dir():
|
||||
for batch_dir in sorted(inbox_lane.iterdir()):
|
||||
if batch_dir.is_dir() and not batch_dir.name.startswith("."):
|
||||
report["batches"].append(
|
||||
enrich_batch(
|
||||
batch_dir,
|
||||
project="lane",
|
||||
task=None,
|
||||
pack=None,
|
||||
batch=batch_dir.name,
|
||||
location="inbox",
|
||||
)
|
||||
)
|
||||
|
||||
report["projects"][pname] = proj
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def _parse_class_summary(text: str) -> dict[str, dict[str, int]]:
|
||||
"""解析 dataset_class_summary.txt 按 task 的类统计。"""
|
||||
by_task: dict[str, dict[str, int]] = {}
|
||||
current_task: str | None = None
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.endswith(":") and " " not in line.rstrip(":"):
|
||||
current_task = line.rstrip(":")
|
||||
by_task.setdefault(current_task, {})
|
||||
continue
|
||||
if current_task and ":" in line:
|
||||
parts = line.split(":", 1)
|
||||
cls_name = parts[0].strip()
|
||||
try:
|
||||
count = int(re.search(r"\d+", parts[1]).group()) # type: ignore
|
||||
by_task[current_task][cls_name] = count
|
||||
except (AttributeError, ValueError):
|
||||
pass
|
||||
return by_task
|
||||
|
||||
|
||||
def _class_name_map(tcfg: dict[str, Any]) -> dict[int, str]:
|
||||
names = tcfg.get("names")
|
||||
if isinstance(names, list):
|
||||
return {idx: str(name) for idx, name in enumerate(names)}
|
||||
if isinstance(names, dict):
|
||||
out: dict[int, str] = {}
|
||||
for k, v in names.items():
|
||||
try:
|
||||
out[int(k)] = str(v)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return out
|
||||
return {}
|
||||
|
||||
|
||||
def _count_images_in_dir(img_dir: Path) -> int:
|
||||
if not img_dir.is_dir():
|
||||
return 0
|
||||
total = 0
|
||||
try:
|
||||
with os.scandir(img_dir) as it:
|
||||
for entry in it:
|
||||
if not entry.is_file(follow_symlinks=False):
|
||||
continue
|
||||
if Path(entry.name).suffix.lower() in IMAGE_EXTS:
|
||||
total += 1
|
||||
except OSError:
|
||||
return 0
|
||||
return total
|
||||
|
||||
|
||||
def _count_split_images(task_data: Path) -> dict[str, int]:
|
||||
counts = {
|
||||
"train": _count_images_in_dir(task_data / "images" / "train"),
|
||||
"val": _count_images_in_dir(task_data / "images" / "val"),
|
||||
"test": _count_images_in_dir(task_data / "images" / "test"),
|
||||
}
|
||||
if sum(counts.values()) == 0:
|
||||
flat = _count_images_in_dir(task_data / "images")
|
||||
if flat:
|
||||
counts["train"] = flat
|
||||
return counts
|
||||
|
||||
|
||||
def _iter_label_files(label_dirs: list[Path]):
|
||||
for label_dir in label_dirs:
|
||||
if not label_dir.is_dir():
|
||||
continue
|
||||
try:
|
||||
with os.scandir(label_dir) as it:
|
||||
stack = [entry.path for entry in it if entry.is_dir(follow_symlinks=False)]
|
||||
files = [entry.path for entry in it if entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt")]
|
||||
except OSError:
|
||||
continue
|
||||
for fp in files:
|
||||
yield Path(fp)
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
try:
|
||||
with os.scandir(current) as it:
|
||||
for entry in it:
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
stack.append(entry.path)
|
||||
elif entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt"):
|
||||
yield Path(entry.path)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
|
||||
def _label_dirs_for_task(task_data: Path) -> list[Path]:
|
||||
return [task_data / "labels" / "train", task_data / "labels" / "val", task_data / "labels"]
|
||||
|
||||
|
||||
def _parse_bbox_wh(parts: list[str]) -> list[float] | None:
|
||||
if len(parts) < 5:
|
||||
return None
|
||||
try:
|
||||
w = float(parts[3])
|
||||
h = float(parts[4])
|
||||
if 0.0 < w <= 1.0 and 0.0 < h <= 1.0:
|
||||
return [round(w, 6), round(h, 6)]
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _collect_bbox_points_sample(task_data: Path, *, max_points: int = MAX_BBOX_POINTS_PER_PACK) -> list[list[float]]:
|
||||
"""Lightweight sample for scatter plot; does not scan images."""
|
||||
bbox_points: list[list[float]] = []
|
||||
remaining_files = MAX_LABEL_FILES_PER_PACK
|
||||
for txt in _iter_label_files(_label_dirs_for_task(task_data)):
|
||||
if len(bbox_points) >= max_points or remaining_files <= 0:
|
||||
break
|
||||
remaining_files -= 1
|
||||
try:
|
||||
for line in txt.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
if len(bbox_points) >= max_points:
|
||||
break
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
wh = _parse_bbox_wh(line.split())
|
||||
if wh:
|
||||
bbox_points.append(wh)
|
||||
except OSError:
|
||||
continue
|
||||
return bbox_points
|
||||
|
||||
|
||||
def _collect_pack_label_distribution(task_data: Path, tcfg: dict[str, Any]) -> dict[str, Any]:
|
||||
label_dirs = _label_dirs_for_task(task_data)
|
||||
class_counts: dict[int, int] = {}
|
||||
bbox_points: list[list[float]] = []
|
||||
parsed_files = 0
|
||||
sampled = False
|
||||
remaining = MAX_LABEL_FILES_PER_PACK
|
||||
for txt in _iter_label_files(label_dirs):
|
||||
if remaining <= 0:
|
||||
sampled = True
|
||||
break
|
||||
remaining -= 1
|
||||
parsed_files += 1
|
||||
try:
|
||||
for line in txt.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
cls_token = line.split(maxsplit=1)[0]
|
||||
cls_id = int(float(cls_token))
|
||||
class_counts[cls_id] = class_counts.get(cls_id, 0) + 1
|
||||
parts = line.split()
|
||||
if len(bbox_points) < MAX_BBOX_POINTS_PER_PACK:
|
||||
wh = _parse_bbox_wh(parts)
|
||||
if wh:
|
||||
bbox_points.append(wh)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
name_map = _class_name_map(tcfg)
|
||||
by_name: dict[str, int] = {}
|
||||
for cls_id, cnt in sorted(class_counts.items(), key=lambda x: x[1], reverse=True):
|
||||
key = name_map.get(cls_id, f"class_{cls_id}")
|
||||
by_name[key] = cnt
|
||||
return {
|
||||
"class_counts": by_name,
|
||||
"label_files": parsed_files,
|
||||
"sampled": sampled,
|
||||
"total_boxes": sum(class_counts.values()),
|
||||
"bbox_points": bbox_points,
|
||||
}
|
||||
|
||||
|
||||
def _histogram(values: list[float], bins: list[float]) -> list[dict[str, float]]:
|
||||
if len(bins) < 2:
|
||||
return []
|
||||
counts = [0] * (len(bins) - 1)
|
||||
for v in values:
|
||||
for i in range(len(bins) - 1):
|
||||
lo, hi = bins[i], bins[i + 1]
|
||||
if (v >= lo and v < hi) or (i == len(bins) - 2 and v >= lo and v <= hi):
|
||||
counts[i] += 1
|
||||
break
|
||||
return [
|
||||
{"left": bins[i], "right": bins[i + 1], "count": counts[i]}
|
||||
for i in range(len(counts))
|
||||
]
|
||||
|
||||
|
||||
def _extract_lane_mask_stats(mask_path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
from PIL import Image # type: ignore
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
try:
|
||||
img = Image.open(mask_path).convert("L")
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
w, h = img.size
|
||||
pix = img.load()
|
||||
if pix is None:
|
||||
return None
|
||||
|
||||
lane_bins: dict[int, list[dict[str, float]]] = {}
|
||||
present_ids: set[int] = set()
|
||||
for y in range(h):
|
||||
by_id: dict[int, tuple[int, int]] = {}
|
||||
for x in range(w):
|
||||
lane_id = int(pix[x, y])
|
||||
if lane_id <= 0:
|
||||
continue
|
||||
present_ids.add(lane_id)
|
||||
if lane_id not in by_id:
|
||||
by_id[lane_id] = (x, x)
|
||||
else:
|
||||
mn, mx = by_id[lane_id]
|
||||
if x < mn:
|
||||
mn = x
|
||||
if x > mx:
|
||||
mx = x
|
||||
by_id[lane_id] = (mn, mx)
|
||||
if not by_id:
|
||||
continue
|
||||
y_bin = min(LANE_Y_BINS - 1, int((y / max(1, h - 1)) * LANE_Y_BINS))
|
||||
for lane_id, (mn, mx) in by_id.items():
|
||||
bucket = lane_bins.setdefault(lane_id, [dict(min_x=1e9, max_x=-1e9, count=0) for _ in range(LANE_Y_BINS)])
|
||||
cur = bucket[y_bin]
|
||||
cur["min_x"] = min(cur["min_x"], float(mn))
|
||||
cur["max_x"] = max(cur["max_x"], float(mx))
|
||||
cur["count"] += 1
|
||||
|
||||
lengths: list[float] = []
|
||||
curvatures: list[float] = []
|
||||
for lane_id in sorted(present_ids):
|
||||
bins = lane_bins.get(lane_id, [])
|
||||
centers: list[tuple[float, float]] = []
|
||||
for i, b in enumerate(bins):
|
||||
if b["count"] <= 0:
|
||||
continue
|
||||
center_x = (b["min_x"] + b["max_x"]) / 2.0
|
||||
center_y = ((i + 0.5) / LANE_Y_BINS) * h
|
||||
centers.append((center_x, center_y))
|
||||
if len(centers) < 2:
|
||||
continue
|
||||
length = 0.0
|
||||
for i in range(1, len(centers)):
|
||||
dx = centers[i][0] - centers[i - 1][0]
|
||||
dy = centers[i][1] - centers[i - 1][1]
|
||||
length += math.sqrt(dx * dx + dy * dy)
|
||||
lengths.append(length)
|
||||
|
||||
if len(centers) >= 3:
|
||||
second_diffs = []
|
||||
xs = [c[0] for c in centers]
|
||||
for i in range(1, len(xs) - 1):
|
||||
second_diffs.append(abs(xs[i + 1] - 2 * xs[i] + xs[i - 1]))
|
||||
if second_diffs:
|
||||
curvatures.append(sum(second_diffs) / len(second_diffs))
|
||||
|
||||
return {
|
||||
"lane_count": len(present_ids),
|
||||
"lengths": lengths,
|
||||
"curvatures": curvatures,
|
||||
}
|
||||
|
||||
|
||||
def _collect_lane_quality(pack_path: Path) -> dict[str, Any]:
|
||||
list_files = [pack_path / "list" / "train_gt.txt", pack_path / "list" / "val_gt.txt"]
|
||||
entries: list[Path] = []
|
||||
for lf in list_files:
|
||||
if not lf.is_file():
|
||||
continue
|
||||
try:
|
||||
for line in lf.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
entries.append(pack_path / parts[1])
|
||||
if len(entries) >= MAX_LANE_MASK_SAMPLES_PER_PACK:
|
||||
break
|
||||
except OSError:
|
||||
continue
|
||||
if len(entries) >= MAX_LANE_MASK_SAMPLES_PER_PACK:
|
||||
break
|
||||
|
||||
lane_counts: list[float] = []
|
||||
lane_lengths: list[float] = []
|
||||
lane_curvatures: list[float] = []
|
||||
processed = 0
|
||||
for ann in entries:
|
||||
s = _extract_lane_mask_stats(ann)
|
||||
if not s:
|
||||
continue
|
||||
processed += 1
|
||||
lane_counts.append(float(s["lane_count"]))
|
||||
lane_lengths.extend(float(x) for x in s["lengths"])
|
||||
lane_curvatures.extend(float(x) for x in s["curvatures"])
|
||||
|
||||
lane_count_hist: dict[str, int] = {}
|
||||
for c in lane_counts:
|
||||
key = str(int(c)) if c < 8 else "8+"
|
||||
lane_count_hist[key] = lane_count_hist.get(key, 0) + 1
|
||||
|
||||
return {
|
||||
"analyzed_frames": processed,
|
||||
"lane_count_hist": lane_count_hist,
|
||||
"length_hist": _histogram(lane_lengths, [0, 60, 120, 180, 240, 320, 420, 560, 760, 1024]),
|
||||
"curvature_hist": _histogram(lane_curvatures, [0, 1, 2, 4, 6, 8, 12, 16, 24, 40]),
|
||||
}
|
||||
|
||||
|
||||
def _catalog_signature(wf: dict) -> dict[str, Any]:
|
||||
return build_catalog_signature(wf, proj_root)
|
||||
|
||||
|
||||
def _build_catalog(wf: dict, *, prefer_reports: bool = True) -> tuple[dict[str, Any], str]:
|
||||
out: dict[str, Any] = {"workspace": str(WORKSPACE), "dms": {}, "lane": {}}
|
||||
build_source = "scan"
|
||||
|
||||
reports = load_dms_reports() if prefer_reports else None
|
||||
report_splits: dict[tuple[str, str], dict[str, int]] = {}
|
||||
report_classes: dict[str, dict[str, int]] = {}
|
||||
if reports:
|
||||
report_splits, report_classes = reports
|
||||
build_source = "reports"
|
||||
|
||||
root = proj_root(wf, "dms")
|
||||
reg_path = root / wf["projects"]["dms"]["registry"]
|
||||
if not reg_path.is_file():
|
||||
out["dms_error"] = f"registry not found: {reg_path}"
|
||||
if report_splits:
|
||||
for (task, pack_name), rep in report_splits.items():
|
||||
entry = out["dms"].setdefault(task, {
|
||||
"type": "unknown",
|
||||
"class_counts": report_classes.get(task, {}),
|
||||
"packs": [],
|
||||
})
|
||||
entry["packs"].append({
|
||||
"name": pack_name,
|
||||
"enabled": False,
|
||||
"train_images": rep.get("train", 0),
|
||||
"val_images": rep.get("val", 0),
|
||||
"test_images": rep.get("test", 0),
|
||||
"class_counts": report_classes.get(task, {}),
|
||||
"label_files": 0,
|
||||
"total_boxes": sum(report_classes.get(task, {}).values()) if task in report_classes else 0,
|
||||
"sampled": True,
|
||||
"bbox_points": [],
|
||||
})
|
||||
reg = {"tasks": {}}
|
||||
else:
|
||||
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
|
||||
try:
|
||||
packs_reg = load_pack_registry("dms", root, wf)
|
||||
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
||||
packs_reg = {"packs": []}
|
||||
summary_path = root / "manifests" / "dataset_class_summary.txt"
|
||||
class_by_task = {}
|
||||
if summary_path.is_file():
|
||||
class_by_task = _parse_class_summary(summary_path.read_text(encoding="utf-8"))
|
||||
|
||||
for task, tcfg in reg.get("tasks", {}).items():
|
||||
entry: dict[str, Any] = {
|
||||
"type": tcfg.get("type"),
|
||||
"nc": tcfg.get("nc"),
|
||||
"names": tcfg.get("names"),
|
||||
"class_counts": class_by_task.get(task, {}),
|
||||
"packs": [],
|
||||
"drop_paths": {
|
||||
"inbox": str((root / "inbox" / task).resolve()),
|
||||
"sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "sources" / "<batch>").resolve()),
|
||||
},
|
||||
}
|
||||
for p in packs_reg.get("packs", []):
|
||||
pack_name = p["name"]
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
task_data = pack_dir / tcfg.get("task_dir", task)
|
||||
rep = report_splits.get((task, pack_name))
|
||||
if rep:
|
||||
split_counts = {"train": rep["train"], "val": rep["val"], "test": rep["test"]}
|
||||
class_counts = report_classes.get(task, class_by_task.get(task, {}))
|
||||
bbox_points = _collect_bbox_points_sample(task_data)
|
||||
label_distribution = {
|
||||
"class_counts": class_counts,
|
||||
"label_files": 0,
|
||||
"sampled": True,
|
||||
"total_boxes": sum(class_counts.values()) if class_counts else 0,
|
||||
"bbox_points": bbox_points,
|
||||
}
|
||||
else:
|
||||
split_counts = _count_split_images(task_data)
|
||||
label_distribution = _collect_pack_label_distribution(task_data, tcfg)
|
||||
if not label_distribution["class_counts"] and task in report_classes:
|
||||
label_distribution["class_counts"] = report_classes[task]
|
||||
entry["packs"].append({
|
||||
"name": pack_name,
|
||||
"path": p.get("path"),
|
||||
"role": p.get("role"),
|
||||
"frozen": p.get("frozen", False),
|
||||
"enabled": pack_name in wf["projects"]["dms"].get("active_packs", []),
|
||||
"train_images": split_counts.get("train", 0),
|
||||
"val_images": split_counts.get("val", 0),
|
||||
"test_images": split_counts.get("test", 0),
|
||||
"class_counts": label_distribution["class_counts"],
|
||||
"label_files": label_distribution["label_files"],
|
||||
"total_boxes": label_distribution["total_boxes"],
|
||||
"sampled": label_distribution["sampled"],
|
||||
"bbox_points": label_distribution["bbox_points"],
|
||||
})
|
||||
if not entry["class_counts"] and task in report_classes:
|
||||
entry["class_counts"] = report_classes[task]
|
||||
out["dms"][task] = entry
|
||||
|
||||
root = proj_root(wf, "lane")
|
||||
try:
|
||||
reg = load_pack_registry("lane", root, wf)
|
||||
except (FileNotFoundError, json.JSONDecodeError, OSError):
|
||||
reg = {"packs": []}
|
||||
for p in reg.get("packs", []):
|
||||
pack_name = p["name"]
|
||||
path = p.get("path", pack_name)
|
||||
pack_path = root / path
|
||||
tg = pack_path / "list" / "train_gt.txt"
|
||||
vg = pack_path / "list" / "val_gt.txt"
|
||||
sg = pack_path / "list" / "test_gt.txt"
|
||||
lane_quality = _collect_lane_quality(pack_path) if LANE_DATA_VIZ_ENABLED else {}
|
||||
out["lane"][pack_name] = {
|
||||
"path": path,
|
||||
"role": p.get("role"),
|
||||
"frozen": p.get("frozen", False),
|
||||
"enabled": pack_name in wf["projects"]["lane"].get("active_packs", []),
|
||||
"train_lines": sum(1 for _ in tg.open(encoding="utf-8")) if tg.is_file() else 0,
|
||||
"val_lines": sum(1 for _ in vg.open(encoding="utf-8")) if vg.is_file() else 0,
|
||||
"test_lines": sum(1 for _ in sg.open(encoding="utf-8")) if sg.is_file() else 0,
|
||||
"drop_path": str(pack_path.resolve()),
|
||||
"add_template": "python as.py add lane --src <archive> --engineer <name> --date YYYYMMDD",
|
||||
"quality": lane_quality,
|
||||
}
|
||||
return out, build_source
|
||||
|
||||
|
||||
def get_catalog(
|
||||
wf: dict | None = None,
|
||||
project: str | None = None,
|
||||
task_or_pack: str | None = None,
|
||||
*,
|
||||
refresh: bool = False,
|
||||
) -> dict:
|
||||
wf = wf or load_wf()
|
||||
sig = _catalog_signature(wf)
|
||||
full_catalog, cache_meta = get_cached_catalog(sig, refresh=refresh)
|
||||
|
||||
if not full_catalog:
|
||||
full_catalog, build_source = _build_catalog(wf, prefer_reports=not refresh)
|
||||
store_catalog_cache(sig, full_catalog, build_source=build_source)
|
||||
cache_meta = {"cached": False, "build_source": build_source}
|
||||
|
||||
result: dict[str, Any]
|
||||
if project == "dms" and task_or_pack:
|
||||
result = {"task": task_or_pack, **(full_catalog.get("dms", {}).get(task_or_pack, {}))}
|
||||
elif project == "lane" and task_or_pack:
|
||||
result = {"pack": task_or_pack, **(full_catalog.get("lane", {}).get(task_or_pack, {}))}
|
||||
elif project in ("dms", "lane"):
|
||||
result = {"workspace": full_catalog.get("workspace", str(WORKSPACE)), project: full_catalog.get(project, {})}
|
||||
else:
|
||||
result = dict(full_catalog)
|
||||
|
||||
if not task_or_pack and project is None:
|
||||
result["_cache"] = cache_meta
|
||||
return result
|
||||
|
||||
|
||||
def warmup_catalog_cache() -> None:
|
||||
"""Background warmup for faster first page load."""
|
||||
try:
|
||||
invalidate_catalog_cache()
|
||||
get_catalog(refresh=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def register_batch(
|
||||
wf: dict | None,
|
||||
project: str,
|
||||
task: str | None,
|
||||
batch: str,
|
||||
*,
|
||||
pack: str | None = None,
|
||||
stage: str = "returned",
|
||||
engineer: str | None = None,
|
||||
location: str = "inbox",
|
||||
) -> dict[str, Any]:
|
||||
wf = wf or load_wf()
|
||||
root = proj_root(wf, project)
|
||||
|
||||
if project == "dms":
|
||||
if not task:
|
||||
raise ValueError("dms register-batch 需要 task")
|
||||
reg = yaml.safe_load((root / wf["projects"]["dms"]["registry"]).read_text(encoding="utf-8"))
|
||||
if task not in reg.get("tasks", {}):
|
||||
raise ValueError(f"未知 task: {task}")
|
||||
tcfg = reg["tasks"][task]
|
||||
if location == "sources":
|
||||
if not pack:
|
||||
raise ValueError("sources 位置需要 --pack")
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack)
|
||||
src_sub = (reg.get("ingest") or {}).get("sources_subdir", "sources")
|
||||
batch_dir = pack_dir / tcfg["task_dir"] / src_sub / batch
|
||||
else:
|
||||
batch_dir = root / "inbox" / task / batch
|
||||
else:
|
||||
if location == "pack" and pack:
|
||||
try:
|
||||
path = resolve_pack("lane", root, wf, pack)
|
||||
batch_dir = root / path
|
||||
except ValueError:
|
||||
batch_dir = root / pack
|
||||
else:
|
||||
batch_dir = root / "inbox" / batch
|
||||
|
||||
if not batch_dir.is_dir():
|
||||
raise FileNotFoundError(f"批次目录不存在: {batch_dir}")
|
||||
|
||||
data = {
|
||||
"schema": "huaxu-batch-v1",
|
||||
"project": project,
|
||||
"task": task,
|
||||
"pack": pack,
|
||||
"batch": batch,
|
||||
"stage": stage,
|
||||
"engineer": engineer,
|
||||
"registered_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
if project == "dms":
|
||||
from as_platform.data.batch import count_images, count_label_files, dms_has_images
|
||||
|
||||
data["format"] = "yolo"
|
||||
data["counts"] = {
|
||||
"images": count_images(batch_dir / "images") + count_images(batch_dir / "images" / "train"),
|
||||
"labels": count_label_files(batch_dir / "labels") + count_label_files(batch_dir / "labels" / "train"),
|
||||
}
|
||||
if not data["counts"]["images"] and dms_has_images(batch_dir):
|
||||
data["counts"]["images"] = 1
|
||||
else:
|
||||
data["format"] = "ufld_archive"
|
||||
tg = batch_dir / "list" / "train_gt.txt"
|
||||
data["counts"] = {"images": 0, "labels": sum(1 for _ in tg.open()) if tg.is_file() else 0}
|
||||
|
||||
meta_path = write_meta(batch_dir, data)
|
||||
invalidate_catalog_cache()
|
||||
return {
|
||||
"ok": True,
|
||||
"meta_path": str(meta_path),
|
||||
"batch": enrich_batch(
|
||||
batch_dir,
|
||||
project=project,
|
||||
task=task,
|
||||
pack=pack,
|
||||
batch=batch,
|
||||
location=location,
|
||||
),
|
||||
}
|
||||
17
platform/as_platform/data/ingest/__init__.py
Normal file
17
platform/as_platform/data/ingest/__init__.py
Normal file
@@ -0,0 +1,17 @@
|
||||
from as_platform.data.ingest.base import IngestContext, IngestAdapter, NormalizedDataset
|
||||
from as_platform.data.ingest.registry import (
|
||||
UnknownFormatError,
|
||||
available_formats,
|
||||
detect_adapter,
|
||||
inspect_uploaded_dataset,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"IngestContext",
|
||||
"IngestAdapter",
|
||||
"NormalizedDataset",
|
||||
"UnknownFormatError",
|
||||
"available_formats",
|
||||
"detect_adapter",
|
||||
"inspect_uploaded_dataset",
|
||||
]
|
||||
46
platform/as_platform/data/ingest/base.py
Normal file
46
platform/as_platform/data/ingest/base.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""Data ingest adapter base abstractions."""
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class IngestContext:
|
||||
project: str
|
||||
task: str | None
|
||||
source_path: Path
|
||||
|
||||
|
||||
@dataclass
|
||||
class NormalizedDataset:
|
||||
format_id: str
|
||||
project: str
|
||||
task: str | None
|
||||
source_path: str
|
||||
split_counts: dict[str, int] = field(default_factory=dict)
|
||||
sample_count: int = 0
|
||||
annotation_count: int = 0
|
||||
artifacts: list[str] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
class IngestAdapter(ABC):
|
||||
"""Adapter interface for task-specific upload formats."""
|
||||
|
||||
format_id: str = "unknown"
|
||||
projects: tuple[str, ...] = ()
|
||||
|
||||
@abstractmethod
|
||||
def can_handle(self, ctx: IngestContext) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
|
||||
raise NotImplementedError
|
||||
88
platform/as_platform/data/ingest/dms_coco.py
Normal file
88
platform/as_platform/data/ingest/dms_coco.py
Normal file
@@ -0,0 +1,88 @@
|
||||
"""DMS COCO-format adapter."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
|
||||
|
||||
COCO_NAMES = ("instances_train.json", "instances_val.json", "instances_test.json", "annotations.json")
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
class DmsCocoAdapter(IngestAdapter):
|
||||
format_id = "dms_coco"
|
||||
projects = ("dms",)
|
||||
|
||||
def _find_coco_files(self, root: Path) -> list[Path]:
|
||||
files: list[Path] = []
|
||||
for name in COCO_NAMES:
|
||||
p = root / "annotations" / name
|
||||
if p.is_file():
|
||||
files.append(p)
|
||||
for name in COCO_NAMES:
|
||||
p = root / name
|
||||
if p.is_file():
|
||||
files.append(p)
|
||||
return files
|
||||
|
||||
def can_handle(self, ctx: IngestContext) -> bool:
|
||||
root = ctx.source_path
|
||||
return len(self._find_coco_files(root)) > 0
|
||||
|
||||
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
|
||||
root = ctx.source_path
|
||||
files = self._find_coco_files(root)
|
||||
split_counts = {"train": 0, "val": 0, "test": 0}
|
||||
ann_count = 0
|
||||
categories: set[str] = set()
|
||||
warnings: list[str] = []
|
||||
for f in files:
|
||||
data = _read_json(f)
|
||||
if not data:
|
||||
warnings.append(f"failed to parse {f.name}")
|
||||
continue
|
||||
images = data.get("images") or []
|
||||
anns = data.get("annotations") or []
|
||||
cats = data.get("categories") or []
|
||||
ann_count += len(anns)
|
||||
for c in cats:
|
||||
name = c.get("name")
|
||||
if isinstance(name, str):
|
||||
categories.add(name)
|
||||
lower = f.name.lower()
|
||||
if "train" in lower:
|
||||
split_counts["train"] += len(images)
|
||||
elif "val" in lower:
|
||||
split_counts["val"] += len(images)
|
||||
elif "test" in lower:
|
||||
split_counts["test"] += len(images)
|
||||
else:
|
||||
split_counts["train"] += len(images)
|
||||
|
||||
return NormalizedDataset(
|
||||
format_id=self.format_id,
|
||||
project=ctx.project,
|
||||
task=ctx.task,
|
||||
source_path=str(root),
|
||||
split_counts=split_counts,
|
||||
sample_count=sum(split_counts.values()),
|
||||
annotation_count=ann_count,
|
||||
artifacts=[self._artifact_name(root, f) for f in files],
|
||||
warnings=warnings,
|
||||
extra={"categories": sorted(categories)},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _artifact_name(root: Path, path: Path) -> str:
|
||||
try:
|
||||
return str(path.relative_to(root))
|
||||
except ValueError:
|
||||
return path.name
|
||||
67
platform/as_platform/data/ingest/dms_yolo.py
Normal file
67
platform/as_platform/data/ingest/dms_yolo.py
Normal file
@@ -0,0 +1,67 @@
|
||||
"""DMS YOLO-style dataset adapter."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
|
||||
|
||||
IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".JPG", ".JPEG", ".PNG"}
|
||||
|
||||
|
||||
def _count_images(path: Path) -> int:
|
||||
if not path.is_dir():
|
||||
return 0
|
||||
return sum(1 for p in path.rglob("*") if p.is_file() and p.suffix in IMG_EXTS)
|
||||
|
||||
|
||||
def _count_txt(path: Path) -> int:
|
||||
if not path.is_dir():
|
||||
return 0
|
||||
return sum(1 for p in path.rglob("*.txt") if p.is_file())
|
||||
|
||||
|
||||
class DmsYoloAdapter(IngestAdapter):
|
||||
format_id = "dms_yolo"
|
||||
projects = ("dms",)
|
||||
|
||||
def can_handle(self, ctx: IngestContext) -> bool:
|
||||
root = ctx.source_path
|
||||
return (
|
||||
(root / "images").is_dir()
|
||||
and (root / "labels").is_dir()
|
||||
) or (
|
||||
(root / "images" / "train").is_dir()
|
||||
and (root / "labels" / "train").is_dir()
|
||||
)
|
||||
|
||||
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
|
||||
root = ctx.source_path
|
||||
train_images = _count_images(root / "images" / "train")
|
||||
val_images = _count_images(root / "images" / "val")
|
||||
test_images = _count_images(root / "images" / "test")
|
||||
if train_images + val_images + test_images == 0:
|
||||
# fallback single-folder dataset
|
||||
train_images = _count_images(root / "images")
|
||||
train_labels = _count_txt(root / "labels" / "train")
|
||||
val_labels = _count_txt(root / "labels" / "val")
|
||||
test_labels = _count_txt(root / "labels" / "test")
|
||||
if train_labels + val_labels + test_labels == 0:
|
||||
train_labels = _count_txt(root / "labels")
|
||||
|
||||
warnings: list[str] = []
|
||||
if train_images == 0:
|
||||
warnings.append("train split has no images")
|
||||
if train_labels == 0:
|
||||
warnings.append("train split has no labels")
|
||||
|
||||
return NormalizedDataset(
|
||||
format_id=self.format_id,
|
||||
project=ctx.project,
|
||||
task=ctx.task,
|
||||
source_path=str(root),
|
||||
split_counts={"train": train_images, "val": val_images, "test": test_images},
|
||||
sample_count=train_images + val_images + test_images,
|
||||
annotation_count=train_labels + val_labels + test_labels,
|
||||
artifacts=["images/", "labels/"],
|
||||
warnings=warnings,
|
||||
)
|
||||
34
platform/as_platform/data/ingest/lane_lines.py
Normal file
34
platform/as_platform/data/ingest/lane_lines.py
Normal file
@@ -0,0 +1,34 @@
|
||||
"""Lane .lines.txt adapter."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
|
||||
|
||||
|
||||
class LaneLinesAdapter(IngestAdapter):
|
||||
format_id = "lane_lines"
|
||||
projects = ("lane",)
|
||||
|
||||
def can_handle(self, ctx: IngestContext) -> bool:
|
||||
root = ctx.source_path
|
||||
return any(root.rglob("*.lines.txt"))
|
||||
|
||||
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
|
||||
root = ctx.source_path
|
||||
line_files = list(root.rglob("*.lines.txt"))
|
||||
split_counts = {"train": len(line_files), "val": 0, "test": 0}
|
||||
warnings: list[str] = []
|
||||
if not line_files:
|
||||
warnings.append("no *.lines.txt found")
|
||||
return NormalizedDataset(
|
||||
format_id=self.format_id,
|
||||
project=ctx.project,
|
||||
task=ctx.task,
|
||||
source_path=str(root),
|
||||
split_counts=split_counts,
|
||||
sample_count=len(line_files),
|
||||
annotation_count=len(line_files),
|
||||
artifacts=["*.lines.txt"],
|
||||
warnings=warnings,
|
||||
)
|
||||
48
platform/as_platform/data/ingest/lane_mask.py
Normal file
48
platform/as_platform/data/ingest/lane_mask.py
Normal file
@@ -0,0 +1,48 @@
|
||||
"""Lane mask + list txt adapter."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
|
||||
|
||||
|
||||
def _line_count(path: Path) -> int:
|
||||
if not path.is_file():
|
||||
return 0
|
||||
try:
|
||||
return sum(1 for _ in path.open(encoding="utf-8", errors="ignore"))
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
class LaneMaskAdapter(IngestAdapter):
|
||||
format_id = "lane_mask"
|
||||
projects = ("lane",)
|
||||
|
||||
def can_handle(self, ctx: IngestContext) -> bool:
|
||||
root = ctx.source_path
|
||||
return (root / "list" / "train_gt.txt").is_file() or (root / "train_val_gt.txt").is_file()
|
||||
|
||||
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
|
||||
root = ctx.source_path
|
||||
train = _line_count(root / "list" / "train_gt.txt")
|
||||
val = _line_count(root / "list" / "val_gt.txt")
|
||||
test = _line_count(root / "list" / "test_gt.txt")
|
||||
if train == 0 and (root / "train_val_gt.txt").is_file():
|
||||
train = _line_count(root / "train_val_gt.txt")
|
||||
|
||||
warnings: list[str] = []
|
||||
if train == 0:
|
||||
warnings.append("train split list is empty")
|
||||
|
||||
return NormalizedDataset(
|
||||
format_id=self.format_id,
|
||||
project=ctx.project,
|
||||
task=ctx.task,
|
||||
source_path=str(root),
|
||||
split_counts={"train": train, "val": val, "test": test},
|
||||
sample_count=train + val + test,
|
||||
annotation_count=train + val + test,
|
||||
artifacts=["list/train_gt.txt", "list/val_gt.txt", "list/test_gt.txt"],
|
||||
warnings=warnings,
|
||||
)
|
||||
49
platform/as_platform/data/ingest/registry.py
Normal file
49
platform/as_platform/data/ingest/registry.py
Normal file
@@ -0,0 +1,49 @@
|
||||
"""Adapter registry and auto detection for uploaded datasets."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
|
||||
from as_platform.data.ingest.dms_coco import DmsCocoAdapter
|
||||
from as_platform.data.ingest.dms_yolo import DmsYoloAdapter
|
||||
from as_platform.data.ingest.lane_lines import LaneLinesAdapter
|
||||
from as_platform.data.ingest.lane_mask import LaneMaskAdapter
|
||||
|
||||
|
||||
class UnknownFormatError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
ADAPTERS: tuple[IngestAdapter, ...] = (
|
||||
DmsYoloAdapter(),
|
||||
DmsCocoAdapter(),
|
||||
LaneMaskAdapter(),
|
||||
LaneLinesAdapter(),
|
||||
)
|
||||
|
||||
|
||||
def available_formats(project: str) -> list[str]:
|
||||
return [a.format_id for a in ADAPTERS if project in a.projects]
|
||||
|
||||
|
||||
def detect_adapter(ctx: IngestContext) -> IngestAdapter:
|
||||
for adapter in ADAPTERS:
|
||||
if ctx.project not in adapter.projects:
|
||||
continue
|
||||
if adapter.can_handle(ctx):
|
||||
return adapter
|
||||
raise UnknownFormatError(
|
||||
f"unable to detect format for project={ctx.project}, task={ctx.task}, "
|
||||
f"source={ctx.source_path}. supported={available_formats(ctx.project)}"
|
||||
)
|
||||
|
||||
|
||||
def inspect_uploaded_dataset(project: str, task: str | None, source_path: str | Path) -> NormalizedDataset:
|
||||
ctx = IngestContext(project=project, task=task, source_path=Path(source_path).resolve())
|
||||
if not ctx.source_path.exists():
|
||||
raise FileNotFoundError(f"source path not found: {ctx.source_path}")
|
||||
adapter = detect_adapter(ctx)
|
||||
out = adapter.inspect(ctx)
|
||||
# Ensure adapter id is always reflected in output.
|
||||
out.format_id = adapter.format_id
|
||||
return out
|
||||
179
platform/as_platform/data/lake.py
Normal file
179
platform/as_platform/data/lake.py
Normal file
@@ -0,0 +1,179 @@
|
||||
"""Upload candidate lifecycle for data-lake ingestion."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import tarfile
|
||||
import uuid
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO
|
||||
|
||||
from as_platform.config import MANIFESTS
|
||||
from as_platform.data.catalog_cache import invalidate_catalog_cache
|
||||
from as_platform.data.ingest import inspect_uploaded_dataset
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import DatasetCandidate
|
||||
|
||||
LAKE_ROOT = MANIFESTS / "lake"
|
||||
UPLOAD_ROOT = LAKE_ROOT / "uploads"
|
||||
STAGING_ROOT = LAKE_ROOT / "staging"
|
||||
REPORT_ROOT = LAKE_ROOT / "reports"
|
||||
|
||||
|
||||
def _new_candidate_id() -> str:
|
||||
return f"cand-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def _candidate_dirs(candidate_id: str) -> tuple[Path, Path, Path]:
|
||||
upload_dir = UPLOAD_ROOT / candidate_id
|
||||
staging_dir = STAGING_ROOT / candidate_id
|
||||
report_file = REPORT_ROOT / f"{candidate_id}.json"
|
||||
return upload_dir, staging_dir, report_file
|
||||
|
||||
|
||||
def create_uploaded_candidate(
|
||||
*,
|
||||
project: str,
|
||||
task: str | None,
|
||||
original_name: str,
|
||||
upload_size_bytes: int,
|
||||
submitted_by_name: str | None,
|
||||
submitted_by_user_id: int | None,
|
||||
) -> dict:
|
||||
candidate_id = _new_candidate_id()
|
||||
upload_dir, _, _ = _candidate_dirs(candidate_id)
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
upload_path = upload_dir / original_name
|
||||
with session_scope() as db:
|
||||
rec = DatasetCandidate(
|
||||
id=candidate_id,
|
||||
project=project,
|
||||
task=task,
|
||||
status="uploaded",
|
||||
source_type="upload",
|
||||
original_name=original_name,
|
||||
upload_path=str(upload_path),
|
||||
upload_size_bytes=upload_size_bytes,
|
||||
submitted_by_name=submitted_by_name,
|
||||
submitted_by_user_id=submitted_by_user_id,
|
||||
)
|
||||
db.add(rec)
|
||||
db.flush()
|
||||
return rec.to_dict()
|
||||
|
||||
|
||||
def write_candidate_upload(candidate_id: str, stream: BinaryIO, chunk_size: int = 1024 * 1024) -> str:
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if not rec:
|
||||
raise ValueError(f"candidate not found: {candidate_id}")
|
||||
path = Path(rec.upload_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("wb") as f:
|
||||
while True:
|
||||
chunk = stream.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
rec.upload_size_bytes = path.stat().st_size
|
||||
db.flush()
|
||||
return str(path)
|
||||
|
||||
|
||||
def list_candidates(limit: int = 100) -> list[dict]:
|
||||
with session_scope() as db:
|
||||
rows = (
|
||||
db.query(DatasetCandidate)
|
||||
.order_by(DatasetCandidate.created_at.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return [r.to_dict() for r in rows]
|
||||
|
||||
|
||||
def get_candidate(candidate_id: str) -> dict | None:
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
return rec.to_dict() if rec else None
|
||||
|
||||
|
||||
def link_candidate_analysis_job(candidate_id: str, job_id: str) -> None:
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if not rec:
|
||||
raise ValueError(f"candidate not found: {candidate_id}")
|
||||
rec.analysis_job_id = job_id
|
||||
db.flush()
|
||||
|
||||
|
||||
def _extract_to_staging(upload_path: Path, staging_dir: Path) -> Path:
|
||||
if staging_dir.exists():
|
||||
shutil.rmtree(staging_dir)
|
||||
staging_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
name = upload_path.name.lower()
|
||||
if name.endswith(".zip"):
|
||||
with zipfile.ZipFile(upload_path, "r") as zf:
|
||||
zf.extractall(staging_dir)
|
||||
elif name.endswith(".tar") or name.endswith(".tar.gz") or name.endswith(".tgz"):
|
||||
with tarfile.open(upload_path, "r:*") as tf:
|
||||
tf.extractall(staging_dir)
|
||||
else:
|
||||
raise ValueError(f"unsupported archive format: {upload_path.name}")
|
||||
|
||||
subdirs = [p for p in staging_dir.iterdir() if p.is_dir()]
|
||||
files = [p for p in staging_dir.iterdir() if p.is_file()]
|
||||
if len(subdirs) == 1 and not files:
|
||||
return subdirs[0]
|
||||
return staging_dir
|
||||
|
||||
|
||||
def analyze_uploaded_candidate(candidate_id: str) -> dict:
|
||||
upload_dir, staging_dir, report_file = _candidate_dirs(candidate_id)
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if not rec:
|
||||
raise ValueError(f"candidate not found: {candidate_id}")
|
||||
rec.status = "analyzing"
|
||||
rec.error_message = None
|
||||
db.flush()
|
||||
project = rec.project
|
||||
task = rec.task
|
||||
upload_path = Path(rec.upload_path)
|
||||
|
||||
if not upload_path.is_file():
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if rec:
|
||||
rec.status = "failed"
|
||||
rec.error_message = f"upload file missing: {upload_path}"
|
||||
raise FileNotFoundError(f"upload file missing: {upload_path}")
|
||||
|
||||
try:
|
||||
dataset_root = _extract_to_staging(upload_path, staging_dir)
|
||||
normalized = inspect_uploaded_dataset(project, task, dataset_root)
|
||||
report_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_file.write_text(json.dumps(normalized.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if not rec:
|
||||
raise ValueError(f"candidate not found during finalize: {candidate_id}")
|
||||
rec.status = "analyzed"
|
||||
rec.analyzed_source_path = str(dataset_root)
|
||||
rec.format_id = normalized.format_id
|
||||
rec.set_split_counts(normalized.split_counts)
|
||||
rec.set_quality(normalized.to_dict())
|
||||
rec.error_message = None
|
||||
db.flush()
|
||||
invalidate_catalog_cache()
|
||||
return normalized.to_dict()
|
||||
except Exception as e:
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if rec:
|
||||
rec.status = "failed"
|
||||
rec.error_message = str(e)
|
||||
db.flush()
|
||||
raise
|
||||
38
platform/as_platform/data/organize.py
Normal file
38
platform/as_platform/data/organize.py
Normal file
@@ -0,0 +1,38 @@
|
||||
"""数据整理:校验摘要写入 batch.meta。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.data.batch import META_FILENAME, count_images, count_label_files, read_meta, write_meta
|
||||
|
||||
|
||||
def organize_batch(batch_dir: Path, *, task: str | None = None) -> dict[str, Any]:
|
||||
"""生成整理报告并合并进 batch.meta.yaml。"""
|
||||
batch_dir = batch_dir.resolve()
|
||||
if not batch_dir.is_dir():
|
||||
raise FileNotFoundError(batch_dir)
|
||||
|
||||
images = count_images(batch_dir / "images") + count_images(batch_dir / "images" / "train")
|
||||
labels = count_label_files(batch_dir / "labels") + count_label_files(batch_dir / "labels" / "train")
|
||||
|
||||
report: dict[str, Any] = {
|
||||
"task": task,
|
||||
"images": images,
|
||||
"labels": labels,
|
||||
"pair_ratio": round(labels / images, 3) if images else 0,
|
||||
"ready_for_ingest": images > 0 and labels > 0,
|
||||
"issues": [],
|
||||
}
|
||||
if images and not labels:
|
||||
report["issues"].append("missing_labels")
|
||||
if labels and not images:
|
||||
report["issues"].append("missing_images")
|
||||
|
||||
meta = read_meta(batch_dir) or {}
|
||||
meta["organize_report"] = report
|
||||
meta.setdefault("counts", {})
|
||||
meta["counts"]["images"] = images
|
||||
meta["counts"]["labels"] = labels
|
||||
write_meta(batch_dir, meta)
|
||||
return report
|
||||
0
platform/as_platform/db/__init__.py
Normal file
0
platform/as_platform/db/__init__.py
Normal file
60
platform/as_platform/db/engine.py
Normal file
60
platform/as_platform/db/engine.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""SQLAlchemy 引擎与会话。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Generator
|
||||
|
||||
from sqlalchemy import create_engine, event, text
|
||||
from sqlalchemy.orm import Session, declarative_base, sessionmaker
|
||||
|
||||
from as_platform.config import DATABASE_URL, IS_POSTGRES, IS_SQLITE
|
||||
|
||||
connect_args: dict = {}
|
||||
engine_kwargs: dict = {"future": True}
|
||||
|
||||
if IS_SQLITE:
|
||||
connect_args["check_same_thread"] = False
|
||||
elif IS_POSTGRES:
|
||||
engine_kwargs.update(pool_pre_ping=True, pool_size=5, max_overflow=10)
|
||||
|
||||
engine = create_engine(DATABASE_URL, connect_args=connect_args, **engine_kwargs)
|
||||
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False, future=True)
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
@event.listens_for(engine, "connect")
|
||||
def _on_connect(dbapi_conn, _):
|
||||
if IS_SQLITE:
|
||||
cursor = dbapi_conn.cursor()
|
||||
cursor.execute("PRAGMA foreign_keys=ON")
|
||||
cursor.close()
|
||||
|
||||
|
||||
def check_connection() -> bool:
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1"))
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
@contextmanager
|
||||
def session_scope() -> Generator[Session, None, None]:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def get_db() -> Generator[Session, None, None]:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
242
platform/as_platform/db/init_db.py
Normal file
242
platform/as_platform/db/init_db.py
Normal file
@@ -0,0 +1,242 @@
|
||||
"""数据库初始化、角色权限种子、jsonl 迁移。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from as_platform.config import (
|
||||
APPROVAL_QUEUE,
|
||||
FEISHU_ADMIN_DEPARTMENT_IDS,
|
||||
FEISHU_ADMIN_OPEN_IDS,
|
||||
IS_POSTGRES,
|
||||
IS_SQLITE,
|
||||
JOB_LOG,
|
||||
MANIFESTS,
|
||||
)
|
||||
from as_platform.db.engine import Base, engine, session_scope
|
||||
from as_platform.db.models import Approval, Job, Permission, Role, User
|
||||
|
||||
# 角色 → 权限
|
||||
ROLE_DEFS: dict[str, tuple[str, list[str]]] = {
|
||||
"admin": ("管理员", ["*"]),
|
||||
"reviewer": ("审核员", [
|
||||
"read:catalog", "read:pending", "read:jobs", "read:audit",
|
||||
"write:approval_review",
|
||||
]),
|
||||
"engineer": ("算法工程师", [
|
||||
"read:catalog", "read:pending", "read:jobs", "read:audit",
|
||||
"write:approval_submit",
|
||||
]),
|
||||
"labeler": ("标注协调", [
|
||||
"read:catalog", "read:pending", "write:approval_submit:register",
|
||||
]),
|
||||
"viewer": ("只读访客", ["read:catalog", "read:pending"]),
|
||||
}
|
||||
|
||||
PERMISSION_NAMES: dict[str, str] = {
|
||||
"*": "全部权限",
|
||||
"read:catalog": "查看数据目录",
|
||||
"read:pending": "查看送标/批次",
|
||||
"read:jobs": "查看 Job 队列",
|
||||
"read:audit": "查看审核记录",
|
||||
"write:approval_submit": "提交审核(训练/build 等)",
|
||||
"write:approval_submit:register": "提交批次登记审核",
|
||||
"write:approval_review": "批准/驳回审核",
|
||||
"admin:users": "用户与角色管理",
|
||||
}
|
||||
|
||||
|
||||
def init_database() -> None:
|
||||
MANIFESTS.mkdir(parents=True, exist_ok=True)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
with session_scope() as db:
|
||||
_ensure_user_columns(db)
|
||||
_seed_roles_permissions(db)
|
||||
_import_jsonl_if_empty(db)
|
||||
_sync_postgres_sequences(db)
|
||||
|
||||
|
||||
def _seed_roles_permissions(db: Session) -> None:
|
||||
perm_map: dict[str, Permission] = {}
|
||||
for code, name in PERMISSION_NAMES.items():
|
||||
p = db.query(Permission).filter_by(code=code).first()
|
||||
if not p:
|
||||
p = Permission(code=code, name=name)
|
||||
db.add(p)
|
||||
db.flush()
|
||||
perm_map[code] = p
|
||||
|
||||
for role_code, (role_name, perm_codes) in ROLE_DEFS.items():
|
||||
role = db.query(Role).filter_by(code=role_code).first()
|
||||
if not role:
|
||||
role = Role(code=role_code, name=role_name)
|
||||
db.add(role)
|
||||
db.flush()
|
||||
role.permissions = [perm_map[c] for c in perm_codes if c in perm_map]
|
||||
|
||||
|
||||
def _import_jsonl_if_empty(db: Session) -> None:
|
||||
if db.query(Approval).count() == 0 and APPROVAL_QUEUE.is_file():
|
||||
for line in APPROVAL_QUEUE.read_text(encoding="utf-8").strip().splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if db.get(Approval, rec.get("id")):
|
||||
continue
|
||||
a = Approval(
|
||||
id=rec["id"],
|
||||
status=rec.get("status", "pending"),
|
||||
action=rec["action"],
|
||||
action_label=rec.get("action_label"),
|
||||
note=rec.get("note"),
|
||||
submitted_by_name=rec.get("submitted_by"),
|
||||
reviewed_by_name=rec.get("reviewed_by"),
|
||||
review_comment=rec.get("review_comment"),
|
||||
job_id=rec.get("job_id"),
|
||||
)
|
||||
a.set_params(rec.get("params") or {})
|
||||
if rec.get("result"):
|
||||
a.set_result(rec["result"])
|
||||
db.add(a)
|
||||
|
||||
if db.query(Job).count() == 0 and JOB_LOG.is_file():
|
||||
for line in JOB_LOG.read_text(encoding="utf-8").strip().splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if db.get(Job, rec.get("id")):
|
||||
continue
|
||||
j = Job(
|
||||
id=rec["id"],
|
||||
status=rec.get("status", "queued"),
|
||||
action=rec["action"],
|
||||
approval_id=rec.get("approval_id"),
|
||||
)
|
||||
j.set_params(rec.get("params") or {})
|
||||
if rec.get("result"):
|
||||
j.set_result(rec["result"])
|
||||
db.add(j)
|
||||
|
||||
|
||||
def assign_default_role(db: Session, user: User) -> None:
|
||||
"""新用户默认角色;支持 open_id / 部门白名单自动 admin。"""
|
||||
user_dept_ids = set(user.feishu_department_ids())
|
||||
if user.feishu_open_id and user.feishu_open_id in FEISHU_ADMIN_OPEN_IDS:
|
||||
role_code = "admin"
|
||||
elif user_dept_ids and user_dept_ids.intersection(FEISHU_ADMIN_DEPARTMENT_IDS):
|
||||
role_code = "admin"
|
||||
elif not user.roles:
|
||||
role_code = "engineer"
|
||||
else:
|
||||
return
|
||||
role = db.query(Role).filter_by(code=role_code).first()
|
||||
if role and role not in user.roles:
|
||||
user.roles.append(role)
|
||||
|
||||
|
||||
def user_has_permission(user: User, permission: str) -> bool:
|
||||
if not user or not user.is_active:
|
||||
return False
|
||||
for role in user.roles:
|
||||
for p in role.permissions:
|
||||
if p.code == "*" or p.code == permission:
|
||||
return True
|
||||
# register_batch 专用权限
|
||||
if permission == "write:approval_submit:register" and p.code == "write:approval_submit":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def user_to_dict(user: User) -> dict[str, Any]:
|
||||
return {
|
||||
"id": user.id,
|
||||
"name": user.name,
|
||||
"email": user.email,
|
||||
"avatar_url": user.avatar_url,
|
||||
"feishu": {
|
||||
"open_id": user.feishu_open_id,
|
||||
"union_id": user.feishu_union_id,
|
||||
"user_id": user.feishu_user_id,
|
||||
"tenant_key": user.feishu_tenant_key,
|
||||
"department_ids": user.feishu_department_ids(),
|
||||
},
|
||||
"roles": [{"code": r.code, "name": r.name} for r in user.roles],
|
||||
"permissions": _collect_permissions(user),
|
||||
}
|
||||
|
||||
|
||||
def _collect_permissions(user: User) -> list[str]:
|
||||
codes: set[str] = set()
|
||||
for role in user.roles:
|
||||
for p in role.permissions:
|
||||
codes.add(p.code)
|
||||
return sorted(codes)
|
||||
|
||||
|
||||
def _sync_postgres_sequences(db: Session) -> None:
|
||||
"""修复 PostgreSQL 自增序列与现有数据不一致问题。"""
|
||||
if not IS_POSTGRES:
|
||||
return
|
||||
# 当历史迁移手动插入了 id,sequence 可能还停留在 1,导致 duplicate key。
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT setval(
|
||||
pg_get_serial_sequence('users', 'id'),
|
||||
COALESCE((SELECT MAX(id) FROM users), 1),
|
||||
true
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT setval(
|
||||
pg_get_serial_sequence('roles', 'id'),
|
||||
COALESCE((SELECT MAX(id) FROM roles), 1),
|
||||
true
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT setval(
|
||||
pg_get_serial_sequence('permissions', 'id'),
|
||||
COALESCE((SELECT MAX(id) FROM permissions), 1),
|
||||
true
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _ensure_user_columns(db: Session) -> None:
|
||||
user_columns = {
|
||||
"feishu_user_id": "VARCHAR(64)",
|
||||
"feishu_tenant_key": "VARCHAR(128)",
|
||||
"feishu_department_ids_json": "TEXT",
|
||||
}
|
||||
if IS_POSTGRES:
|
||||
for column, column_type in user_columns.items():
|
||||
db.execute(text(f"ALTER TABLE users ADD COLUMN IF NOT EXISTS {column} {column_type}"))
|
||||
db.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS ix_users_feishu_user_id ON users (feishu_user_id)"))
|
||||
return
|
||||
|
||||
if IS_SQLITE:
|
||||
rows = db.execute(text("PRAGMA table_info(users)")).fetchall()
|
||||
existing = {str(row[1]) for row in rows}
|
||||
for column, column_type in user_columns.items():
|
||||
if column not in existing:
|
||||
db.execute(text(f"ALTER TABLE users ADD COLUMN {column} {column_type}"))
|
||||
243
platform/as_platform/db/models.py
Normal file
243
platform/as_platform/db/models.py
Normal file
@@ -0,0 +1,243 @@
|
||||
"""ORM 模型。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
Column,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Table,
|
||||
Text,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from as_platform.db.engine import Base
|
||||
|
||||
user_roles = Table(
|
||||
"user_roles",
|
||||
Base.metadata,
|
||||
Column("user_id", Integer, ForeignKey("users.id", ondelete="CASCADE"), primary_key=True),
|
||||
Column("role_id", Integer, ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True),
|
||||
)
|
||||
|
||||
role_permissions = Table(
|
||||
"role_permissions",
|
||||
Base.metadata,
|
||||
Column("role_id", Integer, ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True),
|
||||
Column("permission_id", Integer, ForeignKey("permissions.id", ondelete="CASCADE"), primary_key=True),
|
||||
)
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
feishu_open_id = Column(String(64), unique=True, nullable=True, index=True)
|
||||
feishu_union_id = Column(String(64), unique=True, nullable=True, index=True)
|
||||
feishu_user_id = Column(String(64), unique=True, nullable=True, index=True)
|
||||
feishu_tenant_key = Column(String(128), nullable=True)
|
||||
feishu_department_ids_json = Column(Text, nullable=True)
|
||||
name = Column(String(128), nullable=False, default="")
|
||||
email = Column(String(256), nullable=True)
|
||||
avatar_url = Column(String(512), nullable=True)
|
||||
is_active = Column(Boolean, default=True, nullable=False)
|
||||
created_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
roles = relationship("Role", secondary=user_roles, back_populates="users")
|
||||
|
||||
def feishu_department_ids(self) -> list[str]:
|
||||
if not self.feishu_department_ids_json:
|
||||
return []
|
||||
try:
|
||||
data = json.loads(self.feishu_department_ids_json)
|
||||
if isinstance(data, list):
|
||||
return [str(x) for x in data]
|
||||
except Exception:
|
||||
return []
|
||||
return []
|
||||
|
||||
|
||||
class Role(Base):
|
||||
__tablename__ = "roles"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
code = Column(String(32), unique=True, nullable=False)
|
||||
name = Column(String(64), nullable=False)
|
||||
|
||||
users = relationship("User", secondary=user_roles, back_populates="roles")
|
||||
permissions = relationship("Permission", secondary=role_permissions, back_populates="roles")
|
||||
|
||||
|
||||
class Permission(Base):
|
||||
__tablename__ = "permissions"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
code = Column(String(64), unique=True, nullable=False)
|
||||
name = Column(String(128), nullable=False)
|
||||
|
||||
roles = relationship("Role", secondary=role_permissions, back_populates="permissions")
|
||||
|
||||
|
||||
class Approval(Base):
|
||||
__tablename__ = "approvals"
|
||||
|
||||
id = Column(String(64), primary_key=True)
|
||||
status = Column(String(32), nullable=False, default="pending", index=True)
|
||||
action = Column(String(64), nullable=False)
|
||||
action_label = Column(String(128), nullable=True)
|
||||
params_json = Column(Text, nullable=False, default="{}")
|
||||
note = Column(Text, nullable=True)
|
||||
submitted_by_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
|
||||
submitted_by_name = Column(String(128), nullable=True)
|
||||
submitted_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
reviewed_by_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
|
||||
reviewed_by_name = Column(String(128), nullable=True)
|
||||
reviewed_at = Column(DateTime(timezone=True), nullable=True)
|
||||
review_comment = Column(Text, nullable=True)
|
||||
job_id = Column(String(64), nullable=True)
|
||||
executed_at = Column(DateTime(timezone=True), nullable=True)
|
||||
result_json = Column(Text, nullable=True)
|
||||
|
||||
def params(self) -> dict:
|
||||
return json.loads(self.params_json or "{}")
|
||||
|
||||
def set_params(self, data: dict) -> None:
|
||||
self.params_json = json.dumps(data, ensure_ascii=False)
|
||||
|
||||
def result(self) -> dict | None:
|
||||
if not self.result_json:
|
||||
return None
|
||||
return json.loads(self.result_json)
|
||||
|
||||
def set_result(self, data: dict) -> None:
|
||||
self.result_json = json.dumps(data, ensure_ascii=False)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"status": self.status,
|
||||
"action": self.action,
|
||||
"action_label": self.action_label,
|
||||
"params": self.params(),
|
||||
"note": self.note,
|
||||
"submitted_by": self.submitted_by_name,
|
||||
"submitted_by_user_id": self.submitted_by_user_id,
|
||||
"submitted_at": self.submitted_at.isoformat() if self.submitted_at else None,
|
||||
"reviewed_by": self.reviewed_by_name,
|
||||
"reviewed_by_user_id": self.reviewed_by_user_id,
|
||||
"reviewed_at": self.reviewed_at.isoformat() if self.reviewed_at else None,
|
||||
"review_comment": self.review_comment,
|
||||
"job_id": self.job_id,
|
||||
"executed_at": self.executed_at.isoformat() if self.executed_at else None,
|
||||
"result": self.result(),
|
||||
}
|
||||
|
||||
|
||||
class Job(Base):
|
||||
__tablename__ = "jobs"
|
||||
|
||||
id = Column(String(64), primary_key=True)
|
||||
status = Column(String(32), nullable=False, default="queued", index=True)
|
||||
action = Column(String(64), nullable=False)
|
||||
params_json = Column(Text, nullable=False, default="{}")
|
||||
approval_id = Column(String(64), ForeignKey("approvals.id"), nullable=True)
|
||||
created_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
started_at = Column(DateTime(timezone=True), nullable=True)
|
||||
finished_at = Column(DateTime(timezone=True), nullable=True)
|
||||
result_json = Column(Text, nullable=True)
|
||||
|
||||
def params(self) -> dict:
|
||||
return json.loads(self.params_json or "{}")
|
||||
|
||||
def set_params(self, data: dict) -> None:
|
||||
self.params_json = json.dumps(data, ensure_ascii=False)
|
||||
|
||||
def result(self) -> dict | None:
|
||||
if not self.result_json:
|
||||
return None
|
||||
return json.loads(self.result_json)
|
||||
|
||||
def set_result(self, data: dict) -> None:
|
||||
self.result_json = json.dumps(data, ensure_ascii=False)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"status": self.status,
|
||||
"action": self.action,
|
||||
"params": self.params(),
|
||||
"approval_id": self.approval_id,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"started_at": self.started_at.isoformat() if self.started_at else None,
|
||||
"finished_at": self.finished_at.isoformat() if self.finished_at else None,
|
||||
"result": self.result(),
|
||||
}
|
||||
|
||||
|
||||
class DatasetCandidate(Base):
|
||||
__tablename__ = "dataset_candidates"
|
||||
|
||||
id = Column(String(64), primary_key=True)
|
||||
project = Column(String(32), nullable=False, index=True)
|
||||
task = Column(String(64), nullable=True, index=True)
|
||||
status = Column(String(32), nullable=False, default="uploaded", index=True)
|
||||
source_type = Column(String(32), nullable=False, default="upload")
|
||||
original_name = Column(String(255), nullable=True)
|
||||
upload_path = Column(String(1024), nullable=False)
|
||||
analyzed_source_path = Column(String(1024), nullable=True)
|
||||
format_id = Column(String(64), nullable=True)
|
||||
split_counts_json = Column(Text, nullable=False, default="{}")
|
||||
quality_json = Column(Text, nullable=True)
|
||||
error_message = Column(Text, nullable=True)
|
||||
upload_size_bytes = Column(Integer, nullable=False, default=0)
|
||||
submitted_by_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
|
||||
submitted_by_name = Column(String(128), nullable=True)
|
||||
analysis_job_id = Column(String(64), ForeignKey("jobs.id"), nullable=True)
|
||||
created_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
def split_counts(self) -> dict:
|
||||
return json.loads(self.split_counts_json or "{}")
|
||||
|
||||
def set_split_counts(self, data: dict) -> None:
|
||||
self.split_counts_json = json.dumps(data, ensure_ascii=False)
|
||||
|
||||
def quality(self) -> dict | None:
|
||||
if not self.quality_json:
|
||||
return None
|
||||
return json.loads(self.quality_json)
|
||||
|
||||
def set_quality(self, data: dict) -> None:
|
||||
self.quality_json = json.dumps(data, ensure_ascii=False)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"project": self.project,
|
||||
"task": self.task,
|
||||
"status": self.status,
|
||||
"source_type": self.source_type,
|
||||
"original_name": self.original_name,
|
||||
"upload_path": self.upload_path,
|
||||
"analyzed_source_path": self.analyzed_source_path,
|
||||
"format_id": self.format_id,
|
||||
"split_counts": self.split_counts(),
|
||||
"quality": self.quality(),
|
||||
"error_message": self.error_message,
|
||||
"upload_size_bytes": self.upload_size_bytes,
|
||||
"submitted_by_user_id": self.submitted_by_user_id,
|
||||
"submitted_by_name": self.submitted_by_name,
|
||||
"analysis_job_id": self.analysis_job_id,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||||
}
|
||||
0
platform/as_platform/jobs/__init__.py
Normal file
0
platform/as_platform/jobs/__init__.py
Normal file
145
platform/as_platform/jobs/queue.py
Normal file
145
platform/as_platform/jobs/queue.py
Normal file
@@ -0,0 +1,145 @@
|
||||
"""Job 队列(PostgreSQL + 可选 Redis Worker)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import JOB_EXECUTOR
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import Job
|
||||
|
||||
_executor_lock = threading.Lock()
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _new_id() -> str:
|
||||
return f"job-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def enqueue_job(
|
||||
action: str,
|
||||
params: dict[str, Any],
|
||||
*,
|
||||
approval_id: str | None = None,
|
||||
async_run: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
job_id = _new_id()
|
||||
with session_scope() as db:
|
||||
job = Job(
|
||||
id=job_id,
|
||||
status="queued",
|
||||
action=action,
|
||||
approval_id=approval_id,
|
||||
created_at=_now(),
|
||||
)
|
||||
job.set_params(params)
|
||||
db.add(job)
|
||||
|
||||
out = get_job(job_id) or {"id": job_id, "status": "queued", "action": action}
|
||||
|
||||
if not async_run:
|
||||
_run_job(job_id)
|
||||
return get_job(job_id) or out
|
||||
|
||||
if JOB_EXECUTOR == "worker":
|
||||
from as_platform.redis.bus import push_job
|
||||
|
||||
push_job(job_id)
|
||||
return out
|
||||
|
||||
threading.Thread(target=_run_job, args=(job_id,), daemon=True).start()
|
||||
return out
|
||||
|
||||
|
||||
def get_job(job_id: str) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
rec = db.get(Job, job_id)
|
||||
return rec.to_dict() if rec else None
|
||||
|
||||
|
||||
def list_jobs(status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
|
||||
with session_scope() as db:
|
||||
q = db.query(Job).order_by(Job.created_at.desc())
|
||||
if status:
|
||||
q = q.filter(Job.status == status)
|
||||
return [j.to_dict() for j in q.limit(limit).all()]
|
||||
|
||||
|
||||
def _patch(job_id: str, **fields: Any) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
rec = db.get(Job, job_id)
|
||||
if not rec:
|
||||
return None
|
||||
for k, v in fields.items():
|
||||
if k == "result" and isinstance(v, dict):
|
||||
rec.set_result(v)
|
||||
elif hasattr(rec, k):
|
||||
setattr(rec, k, v)
|
||||
db.flush()
|
||||
return rec.to_dict()
|
||||
|
||||
|
||||
def _compact_result(payload: Any) -> dict[str, Any]:
|
||||
if isinstance(payload, dict):
|
||||
out = dict(payload)
|
||||
else:
|
||||
out = {"value": payload}
|
||||
if "ok" not in out:
|
||||
out["ok"] = True
|
||||
for k in ("stdout", "stderr"):
|
||||
if isinstance(out.get(k), str):
|
||||
out[k] = out[k][-8000:]
|
||||
return out
|
||||
|
||||
|
||||
def _run_job(job_id: str) -> None:
|
||||
with _executor_lock:
|
||||
job = get_job(job_id)
|
||||
if not job or job.get("status") not in ("queued",):
|
||||
return
|
||||
_patch(job_id, status="running", started_at=_now())
|
||||
|
||||
from as_platform.agents.trace import trace_span
|
||||
from as_platform.jobs.runner import execute_action
|
||||
from as_platform.redis.bus import publish
|
||||
|
||||
publish("job.started", {"job_id": job_id, "action": job["action"]})
|
||||
|
||||
try:
|
||||
with trace_span("job_start", job_id=job_id, action=job["action"], approval_id=job.get("approval_id")):
|
||||
result = execute_action(job["action"], job.get("params") or {})
|
||||
persisted = _compact_result(result)
|
||||
_patch(
|
||||
job_id,
|
||||
status="succeeded",
|
||||
finished_at=_now(),
|
||||
result=persisted,
|
||||
)
|
||||
publish("job.succeeded", {"job_id": job_id})
|
||||
with trace_span("job_end", job_id=job_id, status="succeeded"):
|
||||
pass
|
||||
_sync_approval(job.get("approval_id"), "executed", persisted)
|
||||
except Exception as e:
|
||||
_patch(job_id, status="failed", finished_at=_now(), result={"ok": False, "error": str(e)})
|
||||
publish("job.failed", {"job_id": job_id, "error": str(e)})
|
||||
with trace_span("job_end", job_id=job_id, status="failed", error=str(e)):
|
||||
pass
|
||||
_sync_approval(job.get("approval_id"), "failed", {"error": str(e)})
|
||||
|
||||
|
||||
def _sync_approval(approval_id: str | None, status: str, result: dict) -> None:
|
||||
if not approval_id:
|
||||
return
|
||||
from as_platform.audit.queue import _update, _now as audit_now
|
||||
|
||||
_update(
|
||||
approval_id,
|
||||
status=status,
|
||||
executed_at=audit_now(),
|
||||
result=result if isinstance(result, dict) and "ok" in result else {"ok": status == "executed", **result},
|
||||
)
|
||||
159
platform/as_platform/jobs/runner.py
Normal file
159
platform/as_platform/jobs/runner.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""执行动作:优先引擎适配器,fallback as.py CLI。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import WORKSPACE, PLATFORM_DIR, LANE_DATA_VIZ_ENABLED
|
||||
|
||||
if str(WORKSPACE) not in sys.path:
|
||||
sys.path.insert(0, str(WORKSPACE))
|
||||
if str(PLATFORM_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(PLATFORM_DIR))
|
||||
|
||||
ML_PY = WORKSPACE / "as.py"
|
||||
AS_PY = ML_PY
|
||||
|
||||
LONG_ACTIONS = {"train_dms", "train_lane", "pipeline_dms", "eval_dms", "eval_lane", "visualize_dms", "visualize_lane"}
|
||||
|
||||
|
||||
def _run_ml(argv: list[str], timeout: int = 7200) -> dict[str, Any]:
|
||||
cmd = [sys.executable, str(ML_PY), *argv]
|
||||
proc = subprocess.run(cmd, cwd=str(WORKSPACE), capture_output=True, text=True, timeout=timeout)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"as.py 失败 (exit {proc.returncode}):\n{proc.stderr or proc.stdout}")
|
||||
return {"ok": True, "stdout": proc.stdout, "stderr": proc.stderr, "command": " ".join(cmd)}
|
||||
|
||||
|
||||
def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
p = params or {}
|
||||
|
||||
if action == "train_dms":
|
||||
track = p.get("track", "platform")
|
||||
if track == "local":
|
||||
from algorithms.dms_yolo.adapter import train_local
|
||||
return train_local(p["task"], p.get("mode", "full"), p.get("config_overrides"))
|
||||
from algorithms.dms_yolo.adapter import train_platform
|
||||
return train_platform(p["task"], p.get("mode", "full"))
|
||||
|
||||
if action == "train_lane":
|
||||
track = p.get("track", "platform")
|
||||
if track == "local":
|
||||
from algorithms.lane_ufld.adapter import train_local
|
||||
return train_local(p.get("config_overrides"))
|
||||
from algorithms.lane_ufld.adapter import train_platform
|
||||
return train_platform()
|
||||
|
||||
if action == "train_dms_legacy":
|
||||
argv = ["train", "dms", p["task"]]
|
||||
if p.get("mode"):
|
||||
argv.extend(["--mode", str(p["mode"])])
|
||||
return _run_ml(argv, timeout=86400)
|
||||
|
||||
if action == "train_lane_legacy":
|
||||
return _run_ml(["train", "lane"], timeout=86400)
|
||||
|
||||
if action == "build_dms":
|
||||
argv = ["build", "dms", p["task"]]
|
||||
if p.get("pack"):
|
||||
argv.extend(["--pack", str(p["pack"])])
|
||||
if p.get("batch"):
|
||||
argv.extend(["--batch", str(p["batch"])])
|
||||
if p.get("all_sources"):
|
||||
argv.append("--all-sources")
|
||||
if p.get("dry_run"):
|
||||
argv.append("--dry-run")
|
||||
if p.get("skip_validate"):
|
||||
argv.append("--skip-validate")
|
||||
if p.get("no_refresh"):
|
||||
argv.append("--no-refresh")
|
||||
return _run_ml(argv)
|
||||
|
||||
if action == "build_lane":
|
||||
return _run_ml(["build", "lane"])
|
||||
|
||||
if action == "enable_pack":
|
||||
return _run_ml(["enable", p["project"], p["pack"]])
|
||||
|
||||
if action == "disable_pack":
|
||||
return _run_ml(["disable", p["project"], p["pack"]])
|
||||
|
||||
if action == "eval_dms":
|
||||
argv = ["eval", "dms", p["task"]]
|
||||
if p.get("save_candidate"):
|
||||
argv.append("--save-candidate")
|
||||
if p.get("weights"):
|
||||
argv.extend(["--weights", str(p["weights"])])
|
||||
return _run_ml(argv, timeout=3600)
|
||||
|
||||
if action == "eval_lane":
|
||||
from algorithms.lane_ufld.adapter import eval_task
|
||||
|
||||
return eval_task(
|
||||
model_path=p.get("model_path"),
|
||||
data_root=p.get("data_root"),
|
||||
test_list=p.get("test_list", "list/test_gt.txt"),
|
||||
)
|
||||
|
||||
if action == "visualize_dms":
|
||||
from algorithms.dms_yolo.adapter import visualize_task
|
||||
|
||||
return visualize_task(
|
||||
p["task"],
|
||||
weights=p.get("weights"),
|
||||
)
|
||||
|
||||
if action == "visualize_lane":
|
||||
if not LANE_DATA_VIZ_ENABLED:
|
||||
raise RuntimeError("车道线数据可视化暂未开放")
|
||||
from algorithms.lane_ufld.adapter import visualize_task
|
||||
|
||||
return visualize_task(
|
||||
model_path=p.get("model_path"),
|
||||
data_root=p.get("data_root"),
|
||||
test_list=p.get("test_list", "list/test_gt.txt"),
|
||||
)
|
||||
|
||||
if action == "promote_dms":
|
||||
argv = ["promote", "dms", p["task"]]
|
||||
if p.get("force"):
|
||||
argv.append("--force")
|
||||
return _run_ml(argv)
|
||||
|
||||
if action == "pipeline_dms":
|
||||
argv = ["pipeline", "dms", p["task"], "--pack", str(p.get("pack", "dms_v2"))]
|
||||
if p.get("batch"):
|
||||
argv.extend(["--batch", str(p["batch"])])
|
||||
if p.get("all_sources"):
|
||||
argv.append("--all-sources")
|
||||
if p.get("train"):
|
||||
argv.append("--train")
|
||||
if p.get("dry_run"):
|
||||
argv.append("--dry-run")
|
||||
return _run_ml(argv, timeout=86400)
|
||||
|
||||
if action == "register_batch":
|
||||
from as_platform.data.core import register_batch
|
||||
|
||||
register_batch(
|
||||
None, p["project"], p.get("task"), p["batch"],
|
||||
pack=p.get("pack"), stage=p.get("stage", "returned"),
|
||||
engineer=p.get("engineer"), location=p.get("location", "inbox"),
|
||||
)
|
||||
return {"ok": True, "stdout": "register_batch ok", "stderr": ""}
|
||||
|
||||
if action == "analyze_uploaded_dataset":
|
||||
from as_platform.data.lake import analyze_uploaded_candidate
|
||||
|
||||
candidate_id = p["candidate_id"]
|
||||
result = analyze_uploaded_candidate(candidate_id)
|
||||
return {
|
||||
"ok": True,
|
||||
"stdout": json.dumps(result, ensure_ascii=False),
|
||||
"stderr": "",
|
||||
"result": result,
|
||||
}
|
||||
|
||||
raise ValueError(f"未实现执行: {action}")
|
||||
0
platform/as_platform/redis/__init__.py
Normal file
0
platform/as_platform/redis/__init__.py
Normal file
51
platform/as_platform/redis/bus.py
Normal file
51
platform/as_platform/redis/bus.py
Normal file
@@ -0,0 +1,51 @@
|
||||
"""Redis 连接与 Job 事件总线。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import JOB_QUEUE_KEY, REDIS_URL
|
||||
|
||||
try:
|
||||
import redis
|
||||
except ImportError:
|
||||
redis = None # type: ignore
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_redis():
|
||||
if not REDIS_URL or redis is None:
|
||||
return None
|
||||
return redis.from_url(REDIS_URL, decode_responses=True)
|
||||
|
||||
|
||||
def ping_redis() -> bool:
|
||||
try:
|
||||
r = get_redis()
|
||||
return bool(r and r.ping())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def publish(event: str, payload: dict[str, Any]) -> None:
|
||||
r = get_redis()
|
||||
if not r:
|
||||
return
|
||||
r.publish("as:events", json.dumps({"event": event, **payload}, ensure_ascii=False))
|
||||
|
||||
|
||||
def push_job(job_id: str) -> None:
|
||||
r = get_redis()
|
||||
if not r:
|
||||
raise RuntimeError("Redis 未配置,无法使用 worker 模式")
|
||||
r.lpush(JOB_QUEUE_KEY, job_id)
|
||||
publish("job.queued", {"job_id": job_id})
|
||||
|
||||
|
||||
def pop_job(timeout: int = 5) -> str | None:
|
||||
r = get_redis()
|
||||
if not r:
|
||||
return None
|
||||
item = r.brpop(JOB_QUEUE_KEY, timeout=timeout)
|
||||
return item[1] if item else None
|
||||
37
platform/as_platform/sdk.py
Normal file
37
platform/as_platform/sdk.py
Normal file
@@ -0,0 +1,37 @@
|
||||
"""华胥平台 Python SDK — CLI / Job / Agent 共用。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from as_platform.audit.queue import (
|
||||
approve_and_execute,
|
||||
get_approval,
|
||||
list_approvals,
|
||||
reject_approval,
|
||||
submit_approval,
|
||||
)
|
||||
from as_platform.config import WORKSPACE
|
||||
from as_platform.data.core import get_catalog, get_pending_report, load_wf, register_batch
|
||||
from as_platform.data.organize import organize_batch
|
||||
from as_platform.jobs.queue import enqueue_job, get_job, list_jobs
|
||||
from as_platform.agents.tools import invoke_tool, TOOL_REGISTRY
|
||||
from as_platform.agents.trace import get_trace, start_trace
|
||||
|
||||
__all__ = [
|
||||
"WORKSPACE",
|
||||
"get_pending_report",
|
||||
"get_catalog",
|
||||
"register_batch",
|
||||
"organize_batch",
|
||||
"load_wf",
|
||||
"submit_approval",
|
||||
"list_approvals",
|
||||
"get_approval",
|
||||
"approve_and_execute",
|
||||
"reject_approval",
|
||||
"enqueue_job",
|
||||
"get_job",
|
||||
"list_jobs",
|
||||
"invoke_tool",
|
||||
"TOOL_REGISTRY",
|
||||
"get_trace",
|
||||
"start_trace",
|
||||
]
|
||||
1
platform/as_platform/training/__init__.py
Normal file
1
platform/as_platform/training/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""训练记录聚合(Job + Approval + 模型 manifest)。"""
|
||||
216
platform/as_platform/training/service.py
Normal file
216
platform/as_platform/training/service.py
Normal file
@@ -0,0 +1,216 @@
|
||||
"""训练记录查询与提交。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from as_platform.audit.queue import ACTION_LABELS, get_approval, submit_approval
|
||||
from as_platform.config import WORKSPACE
|
||||
from as_platform.jobs.queue import get_job, list_jobs
|
||||
|
||||
TRAINING_ACTIONS = frozenset(
|
||||
{
|
||||
"train_dms",
|
||||
"train_lane",
|
||||
"eval_dms",
|
||||
"eval_lane",
|
||||
"promote_dms",
|
||||
"pipeline_dms",
|
||||
"visualize_dms",
|
||||
"visualize_lane",
|
||||
}
|
||||
)
|
||||
|
||||
ACTION_KIND = {
|
||||
"train_dms": "train",
|
||||
"train_lane": "train",
|
||||
"eval_dms": "eval",
|
||||
"eval_lane": "eval",
|
||||
"promote_dms": "promote",
|
||||
"pipeline_dms": "pipeline",
|
||||
"visualize_dms": "visualize",
|
||||
"visualize_lane": "visualize",
|
||||
}
|
||||
|
||||
|
||||
def _project_for_action(action: str) -> str:
|
||||
if action.endswith("_dms") or action.startswith("promote_dms") or action.startswith("pipeline_dms"):
|
||||
return "dms"
|
||||
if "lane" in action:
|
||||
return "lane"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _parse_ts(ts: str | None) -> datetime | None:
|
||||
if not ts:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _duration_sec(job: dict[str, Any]) -> float | None:
|
||||
start = _parse_ts(job.get("started_at") or job.get("created_at"))
|
||||
end = _parse_ts(job.get("finished_at"))
|
||||
if not start or not end:
|
||||
return None
|
||||
return max(0.0, (end - start).total_seconds())
|
||||
|
||||
|
||||
def _extract_weight(job: dict[str, Any]) -> str | None:
|
||||
params = job.get("params") or {}
|
||||
result = job.get("result") or {}
|
||||
for key in ("best_weights", "candidate", "model_path", "weights", "run_dir"):
|
||||
val = result.get(key) or params.get(key)
|
||||
if isinstance(val, str) and val.strip():
|
||||
return val.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _extract_metrics(result: dict[str, Any]) -> dict[str, Any]:
|
||||
if not isinstance(result, dict):
|
||||
return {}
|
||||
metrics: dict[str, Any] = {}
|
||||
for key in ("map50", "map50_95", "map", "delta_map50", "precision", "recall", "f1"):
|
||||
if key in result and result[key] is not None:
|
||||
metrics[key] = result[key]
|
||||
if "metrics" in result and isinstance(result["metrics"], dict):
|
||||
metrics.update(result["metrics"])
|
||||
if "last_eval" in result and isinstance(result["last_eval"], dict):
|
||||
metrics.update(result["last_eval"])
|
||||
return metrics
|
||||
|
||||
|
||||
def enrich_job(job: dict[str, Any]) -> dict[str, Any]:
|
||||
action = job.get("action", "")
|
||||
params = job.get("params") or {}
|
||||
result = job.get("result") or {}
|
||||
approval = get_approval(job["approval_id"]) if job.get("approval_id") else None
|
||||
task = params.get("task")
|
||||
if action == "train_lane" and not task:
|
||||
task = None
|
||||
return {
|
||||
**job,
|
||||
"action_label": ACTION_LABELS.get(action, action),
|
||||
"project": _project_for_action(action),
|
||||
"kind": ACTION_KIND.get(action, "other"),
|
||||
"task": task,
|
||||
"track": params.get("track"),
|
||||
"weight_path": _extract_weight(job),
|
||||
"metrics": _extract_metrics(result),
|
||||
"error": result.get("error") if isinstance(result, dict) else None,
|
||||
"approval": approval,
|
||||
"duration_sec": _duration_sec(job),
|
||||
}
|
||||
|
||||
|
||||
def _summarize(records: list[dict[str, Any]]) -> dict[str, int]:
|
||||
summary = {"total": len(records), "running": 0, "queued": 0, "succeeded": 0, "failed": 0}
|
||||
for rec in records:
|
||||
status = rec.get("status") or ""
|
||||
if status in summary:
|
||||
summary[status] += 1
|
||||
return summary
|
||||
|
||||
|
||||
def list_training_records(
|
||||
*,
|
||||
project: str | None = None,
|
||||
kind: str | None = None,
|
||||
status: str | None = None,
|
||||
task: str | None = None,
|
||||
limit: int = 100,
|
||||
) -> dict[str, Any]:
|
||||
jobs = list_jobs(status=status, limit=500)
|
||||
records: list[dict[str, Any]] = []
|
||||
for job in jobs:
|
||||
if job.get("action") not in TRAINING_ACTIONS:
|
||||
continue
|
||||
rec = enrich_job(job)
|
||||
if project and rec["project"] != project:
|
||||
continue
|
||||
if kind and rec["kind"] != kind:
|
||||
continue
|
||||
if task and rec.get("task") != task:
|
||||
continue
|
||||
records.append(rec)
|
||||
if len(records) >= limit:
|
||||
break
|
||||
return {"items": records, "total": len(records), "summary": _summarize(records)}
|
||||
|
||||
|
||||
def get_training_record(job_id: str) -> dict[str, Any] | None:
|
||||
job = get_job(job_id)
|
||||
if not job or job.get("action") not in TRAINING_ACTIONS:
|
||||
return None
|
||||
return enrich_job(job)
|
||||
|
||||
|
||||
def _read_train_versions() -> dict[str, Any]:
|
||||
path = WORKSPACE / "datasets/dms/manifests/train_versions.yaml"
|
||||
if not path.is_file():
|
||||
return {}
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _read_eval_log(task: str | None = None, limit: int = 30) -> list[dict[str, Any]]:
|
||||
path = WORKSPACE / "datasets/dms/manifests/eval_log.jsonl"
|
||||
if not path.is_file():
|
||||
return []
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
entries: list[dict[str, Any]] = []
|
||||
for line in reversed(lines):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if task and row.get("task") != task:
|
||||
continue
|
||||
entries.append(row)
|
||||
if len(entries) >= limit:
|
||||
break
|
||||
return entries
|
||||
|
||||
|
||||
def get_model_registry(project: str = "dms", task: str | None = None) -> dict[str, Any]:
|
||||
if project != "dms":
|
||||
return {"project": project, "tasks": {}, "eval_history": []}
|
||||
versions = _read_train_versions()
|
||||
if task:
|
||||
task_data = versions.get(task, {})
|
||||
return {
|
||||
"project": "dms",
|
||||
"task": task,
|
||||
"version": task_data,
|
||||
"eval_history": _read_eval_log(task=task),
|
||||
}
|
||||
tasks = {name: data for name, data in versions.items() if isinstance(data, dict)}
|
||||
return {"project": "dms", "tasks": tasks, "eval_history": _read_eval_log(limit=20)}
|
||||
|
||||
|
||||
def create_training_submission(
|
||||
action: str,
|
||||
params: dict[str, Any],
|
||||
*,
|
||||
submitted_by: str | None = None,
|
||||
submitted_by_user_id: int | None = None,
|
||||
note: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if action not in TRAINING_ACTIONS:
|
||||
raise ValueError(f"不支持的动作: {action}")
|
||||
return submit_approval(
|
||||
action,
|
||||
params,
|
||||
submitted_by=submitted_by,
|
||||
submitted_by_user_id=submitted_by_user_id,
|
||||
note=note,
|
||||
)
|
||||
3
platform/web/env.sh
Normal file
3
platform/web/env.sh
Normal file
@@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
# 将 Node.js(~/.local/node)加入 PATH,供 npm run dev 使用
|
||||
export PATH="${HOME}/.local/node/bin:${PATH}"
|
||||
15
platform/web/index.html
Normal file
15
platform/web/index.html
Normal file
@@ -0,0 +1,15 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>HSAP · Huaxu Sentinel Active Safety Platform</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link href="https://fonts.googleapis.com/css2?family=DM+Sans:opsz,wght@9..40,400;9..40,500;9..40,600;9..40,700&family=Noto+Sans+SC:wght@400;500;600;700&display=swap" rel="stylesheet" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
1878
platform/web/package-lock.json
generated
Normal file
1878
platform/web/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
23
platform/web/package.json
Normal file
23
platform/web/package.json
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "huaxu-as-web",
|
||||
"private": true,
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc -b && vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"react-router-dom": "^7.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.0.0",
|
||||
"@types/react-dom": "^19.0.0",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
"typescript": "~5.7.2",
|
||||
"vite": "^6.0.0"
|
||||
}
|
||||
}
|
||||
47
platform/web/src/App.tsx
Normal file
47
platform/web/src/App.tsx
Normal file
@@ -0,0 +1,47 @@
|
||||
import { BrowserRouter, Navigate, Route, Routes } from "react-router-dom";
|
||||
import { AuthProvider } from "./auth/AuthContext";
|
||||
import { RequireAuth } from "./auth/RequireAuth";
|
||||
import { Layout } from "./components/Layout";
|
||||
import { ToastProvider } from "./components/Toast";
|
||||
import { AuthCallbackPage } from "./pages/AuthCallback";
|
||||
import { LoginPage } from "./pages/Login";
|
||||
import { LabelingPage } from "./pages/Labeling";
|
||||
import { CatalogPage } from "./pages/Catalog";
|
||||
import { AuditDetailPage, AuditPage } from "./pages/Audit";
|
||||
import { JobsPage } from "./pages/Jobs";
|
||||
import { TrainingPage } from "./pages/Training";
|
||||
import { LogsPage } from "./pages/Logs";
|
||||
|
||||
export default function App() {
|
||||
return (
|
||||
<ToastProvider>
|
||||
<AuthProvider>
|
||||
<BrowserRouter>
|
||||
<Routes>
|
||||
<Route path="/login" element={<LoginPage />} />
|
||||
<Route path="/auth/callback" element={<AuthCallbackPage />} />
|
||||
<Route
|
||||
path="/"
|
||||
element={
|
||||
<RequireAuth>
|
||||
<Layout />
|
||||
</RequireAuth>
|
||||
}
|
||||
>
|
||||
<Route index element={<Navigate to="/labeling" replace />} />
|
||||
<Route path="labeling" element={<LabelingPage />} />
|
||||
<Route path="catalog" element={<CatalogPage />} />
|
||||
<Route path="audit" element={<AuditPage />} />
|
||||
<Route path="audit/:id" element={<AuditDetailPage />} />
|
||||
<Route path="jobs" element={<JobsPage />} />
|
||||
<Route path="uploads" element={<Navigate to="/labeling" replace />} />
|
||||
<Route path="training" element={<TrainingPage />} />
|
||||
<Route path="iterate" element={<Navigate to="/training" replace />} />
|
||||
<Route path="logs" element={<LogsPage />} />
|
||||
</Route>
|
||||
</Routes>
|
||||
</BrowserRouter>
|
||||
</AuthProvider>
|
||||
</ToastProvider>
|
||||
);
|
||||
}
|
||||
378
platform/web/src/api/client.ts
Normal file
378
platform/web/src/api/client.ts
Normal file
@@ -0,0 +1,378 @@
|
||||
const API_BASE = import.meta.env.VITE_API_BASE || "";
|
||||
|
||||
let _token: string | null = localStorage.getItem("as_access_token");
|
||||
|
||||
function authHeaders(): Record<string, string> {
|
||||
const h: Record<string, string> = { "Content-Type": "application/json" };
|
||||
if (_token) h.Authorization = `Bearer ${_token}`;
|
||||
return h;
|
||||
}
|
||||
|
||||
async function fetchJson<T>(url: string, init?: RequestInit): Promise<T> {
|
||||
const res = await fetch(url, {
|
||||
...init,
|
||||
cache: "no-store",
|
||||
headers: { ...authHeaders(), ...(init?.headers as Record<string, string>) },
|
||||
});
|
||||
if (res.status === 401) throw new Error("UNAUTHORIZED");
|
||||
if (!res.ok) throw new Error(await res.text() || res.statusText);
|
||||
return (await res.json()) as T;
|
||||
}
|
||||
|
||||
async function postJson<T = unknown>(url: string, body?: unknown): Promise<T> {
|
||||
return fetchJson<T>(url, {
|
||||
method: "POST",
|
||||
body: body !== undefined ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function uploadWithProgress(
|
||||
url: string,
|
||||
formData: FormData,
|
||||
onProgress?: (percent: number) => void
|
||||
): Promise<{ candidate: DataCandidate; job: JobRecord }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("POST", url, true);
|
||||
if (_token) xhr.setRequestHeader("Authorization", `Bearer ${_token}`);
|
||||
xhr.upload.onprogress = (evt) => {
|
||||
if (!onProgress || !evt.lengthComputable) return;
|
||||
const pct = Math.round((evt.loaded / evt.total) * 100);
|
||||
onProgress(pct);
|
||||
};
|
||||
xhr.onerror = () => reject(new Error("上传失败"));
|
||||
xhr.onload = () => {
|
||||
if (xhr.status === 401) return reject(new Error("UNAUTHORIZED"));
|
||||
if (xhr.status < 200 || xhr.status >= 300) return reject(new Error(xhr.responseText || `HTTP ${xhr.status}`));
|
||||
try {
|
||||
const parsed = JSON.parse(xhr.responseText || "{}");
|
||||
resolve(parsed);
|
||||
} catch (e) {
|
||||
reject(new Error("上传响应解析失败"));
|
||||
}
|
||||
};
|
||||
xhr.send(formData);
|
||||
});
|
||||
}
|
||||
|
||||
export type AuthUser = {
|
||||
id: number;
|
||||
name: string;
|
||||
email?: string;
|
||||
avatar_url?: string;
|
||||
roles: { code: string; name: string }[];
|
||||
permissions: string[];
|
||||
};
|
||||
|
||||
export const api = {
|
||||
base: API_BASE || window.location.origin,
|
||||
|
||||
setToken(token: string | null) {
|
||||
_token = token;
|
||||
},
|
||||
|
||||
authConfig: () => fetchJson<{ feishu_enabled: boolean; dev_auth_enabled: boolean }>(`${API_BASE}/api/v1/auth/config`),
|
||||
|
||||
me: () => fetchJson<AuthUser>(`${API_BASE}/api/v1/auth/me`),
|
||||
|
||||
devLogin: (name?: string) => postJson<{ access_token: string; user: AuthUser }>(`${API_BASE}/api/v1/auth/dev/login`, { name: name || "开发用户" }),
|
||||
|
||||
health: () =>
|
||||
fetchJson<{
|
||||
status: string;
|
||||
workspace?: string;
|
||||
database?: string;
|
||||
db_connected?: string;
|
||||
redis_connected?: string;
|
||||
}>(`${API_BASE}/api/v1/health`),
|
||||
|
||||
pending: () => fetchJson<PendingReport>(`${API_BASE}/api/v1/pending`),
|
||||
|
||||
catalog: (refresh = false) =>
|
||||
fetchJson<CatalogReport>(`${API_BASE}/api/v1/catalog${refresh ? "?refresh=true" : ""}`),
|
||||
|
||||
listApprovals: (status?: string) => {
|
||||
const q = status ? `?status=${encodeURIComponent(status)}` : "";
|
||||
return fetchJson<{ items: ApprovalRecord[] }>(`${API_BASE}/api/v1/approvals${q}`);
|
||||
},
|
||||
|
||||
getApproval: (id: string) => fetchJson<ApprovalRecord>(`${API_BASE}/api/v1/approvals/${encodeURIComponent(id)}`),
|
||||
|
||||
getApprovalPreview: (id: string) =>
|
||||
fetchJson<AuditPreview>(`${API_BASE}/api/v1/approvals/${encodeURIComponent(id)}/preview`),
|
||||
|
||||
listApprovalImages: (id: string, offset = 0, limit = 60) =>
|
||||
fetchJson<{ total: number; offset: number; limit: number; items: AuditImageItem[] }>(
|
||||
`${API_BASE}/api/v1/approvals/${encodeURIComponent(id)}/images?offset=${offset}&limit=${limit}`
|
||||
),
|
||||
|
||||
fetchApprovalImageBlob: async (approvalId: string, imageId: string, thumb = true): Promise<string> => {
|
||||
const q = thumb ? "?thumb=true" : "?thumb=false";
|
||||
const res = await fetch(
|
||||
`${API_BASE}/api/v1/approvals/${encodeURIComponent(approvalId)}/images/${encodeURIComponent(imageId)}${q}`,
|
||||
{ headers: authHeaders(), cache: "no-store" }
|
||||
);
|
||||
if (!res.ok) throw new Error(await res.text() || res.statusText);
|
||||
return URL.createObjectURL(await res.blob());
|
||||
},
|
||||
|
||||
submitApproval: (action: string, params: Record<string, unknown>, note?: string) =>
|
||||
postJson(`${API_BASE}/api/v1/approvals/submit`, { action, params, note }),
|
||||
|
||||
submitBuildBatch: (body: Record<string, unknown>) =>
|
||||
postJson(`${API_BASE}/api/v1/approvals/submit-build-batch`, body),
|
||||
|
||||
approve: (id: string, comment?: string) =>
|
||||
postJson(`${API_BASE}/api/v1/approvals/${id}/approve`, { comment }),
|
||||
|
||||
reject: (id: string, comment?: string) =>
|
||||
postJson(`${API_BASE}/api/v1/approvals/${id}/reject`, { comment }),
|
||||
|
||||
registerBatch: (body: Record<string, unknown>) =>
|
||||
postJson(`${API_BASE}/api/v1/register-batch`, body),
|
||||
|
||||
listJobs: (status?: string) => {
|
||||
const q = status ? `?status=${encodeURIComponent(status)}` : "";
|
||||
return fetchJson<{ items: JobRecord[] }>(`${API_BASE}/api/v1/jobs${q}`).catch(() => ({ items: [] }));
|
||||
},
|
||||
|
||||
listTrainingRecords: (opts?: {
|
||||
project?: string;
|
||||
kind?: string;
|
||||
status?: string;
|
||||
task?: string;
|
||||
limit?: number;
|
||||
}) => {
|
||||
const params = new URLSearchParams();
|
||||
if (opts?.project) params.set("project", opts.project);
|
||||
if (opts?.kind) params.set("kind", opts.kind);
|
||||
if (opts?.status) params.set("status", opts.status);
|
||||
if (opts?.task) params.set("task", opts.task);
|
||||
if (opts?.limit) params.set("limit", String(opts.limit));
|
||||
const q = params.toString();
|
||||
return fetchJson<{ items: TrainingRecord[]; total: number; summary: TrainingSummary }>(
|
||||
`${API_BASE}/api/v1/training/records${q ? `?${q}` : ""}`
|
||||
);
|
||||
},
|
||||
|
||||
getTrainingRecord: (jobId: string) =>
|
||||
fetchJson<TrainingRecord>(`${API_BASE}/api/v1/training/records/${encodeURIComponent(jobId)}`),
|
||||
|
||||
getModelRegistry: (project = "dms", task?: string) => {
|
||||
const params = new URLSearchParams({ project });
|
||||
if (task) params.set("task", task);
|
||||
return fetchJson<ModelRegistry>(`${API_BASE}/api/v1/training/models?${params.toString()}`);
|
||||
},
|
||||
|
||||
createTrainingRecord: (action: string, params: Record<string, unknown>, note?: string) =>
|
||||
postJson<ApprovalRecord>(`${API_BASE}/api/v1/training/records`, { action, params, note }),
|
||||
|
||||
uploadDatasetFile: (
|
||||
file: File,
|
||||
project: string,
|
||||
task?: string,
|
||||
onProgress?: (percent: number) => void
|
||||
) => {
|
||||
const formData = new FormData();
|
||||
formData.append("project", project);
|
||||
if (task) formData.append("task", task);
|
||||
formData.append("file", file);
|
||||
return uploadWithProgress(`${API_BASE}/api/v1/data/upload/file`, formData, onProgress);
|
||||
},
|
||||
|
||||
listDataCandidates: (limit = 50) =>
|
||||
fetchJson<{ items: DataCandidate[] }>(`${API_BASE}/api/v1/data/candidates?limit=${limit}`),
|
||||
|
||||
getDataCandidate: (candidateId: string) =>
|
||||
fetchJson<DataCandidate>(`${API_BASE}/api/v1/data/candidates/${encodeURIComponent(candidateId)}`),
|
||||
|
||||
inspectUploadPath: (project: "dms" | "lane", sourcePath: string, task?: string) =>
|
||||
postJson<InspectUploadResponse>(`${API_BASE}/api/v1/data/inspect-upload`, {
|
||||
project,
|
||||
task: task || undefined,
|
||||
source_path: sourcePath,
|
||||
}),
|
||||
};
|
||||
|
||||
export type BatchRecord = {
|
||||
project: string;
|
||||
task?: string;
|
||||
batch: string;
|
||||
pack?: string;
|
||||
stage: string;
|
||||
location: string;
|
||||
path?: string;
|
||||
counts?: { images?: number; labels?: number };
|
||||
engineer?: string;
|
||||
format?: string;
|
||||
next_cli?: string;
|
||||
};
|
||||
|
||||
export type PendingReport = {
|
||||
workspace: string;
|
||||
batches: BatchRecord[];
|
||||
projects?: {
|
||||
dms?: {
|
||||
active_packs?: string[];
|
||||
not_enabled?: string[];
|
||||
task_defs?: Record<string, { type: string; nc?: number }>;
|
||||
tasks?: Record<string, { inbox?: unknown[]; sources?: Record<string, unknown[]> }>;
|
||||
recent_ingest?: { task: string; pack?: string; ts?: string; added?: number }[];
|
||||
};
|
||||
lane?: { packs?: Record<string, { path: string; train_lines?: number; enabled?: boolean }> };
|
||||
};
|
||||
};
|
||||
|
||||
export type CatalogReport = {
|
||||
_cache?: {
|
||||
cached?: boolean;
|
||||
cache_source?: string;
|
||||
cache_age_sec?: number;
|
||||
build_source?: string;
|
||||
};
|
||||
dms?: Record<string, {
|
||||
type: string;
|
||||
nc?: number;
|
||||
class_counts?: Record<string, number>;
|
||||
packs?: {
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
train_images?: number;
|
||||
val_images?: number;
|
||||
test_images?: number;
|
||||
class_counts?: Record<string, number>;
|
||||
label_files?: number;
|
||||
total_boxes?: number;
|
||||
sampled?: boolean;
|
||||
bbox_points?: [number, number][];
|
||||
path?: string;
|
||||
role?: string;
|
||||
frozen?: boolean;
|
||||
}[];
|
||||
drop_paths?: { inbox?: string };
|
||||
}>;
|
||||
lane?: Record<string, {
|
||||
path?: string;
|
||||
drop_path?: string;
|
||||
train_lines?: number;
|
||||
val_lines?: number;
|
||||
test_lines?: number;
|
||||
enabled?: boolean;
|
||||
add_template?: string;
|
||||
quality?: {
|
||||
analyzed_frames?: number;
|
||||
lane_count_hist?: Record<string, number>;
|
||||
length_hist?: { left: number; right: number; count: number }[];
|
||||
curvature_hist?: { left: number; right: number; count: number }[];
|
||||
};
|
||||
}>;
|
||||
};
|
||||
|
||||
export type ApprovalRecord = {
|
||||
id: string;
|
||||
action: string;
|
||||
action_label?: string;
|
||||
status: string;
|
||||
params?: Record<string, unknown>;
|
||||
note?: string | null;
|
||||
submitted_by?: string;
|
||||
submitted_at?: string;
|
||||
review_comment?: string;
|
||||
result?: { error?: string; ok?: boolean };
|
||||
};
|
||||
|
||||
export type AuditImageItem = {
|
||||
id: string;
|
||||
batch: string;
|
||||
location: string;
|
||||
split: string;
|
||||
filename: string;
|
||||
has_label: boolean;
|
||||
box_count: number;
|
||||
missing_label: boolean;
|
||||
};
|
||||
|
||||
export type AuditPreview = {
|
||||
approval: ApprovalRecord;
|
||||
scope_label?: string;
|
||||
task?: string;
|
||||
pack?: string;
|
||||
class_names?: Record<number, string>;
|
||||
batches?: { batch?: string; location?: string; path?: string; exists?: boolean }[];
|
||||
};
|
||||
|
||||
export type JobRecord = {
|
||||
id: string;
|
||||
action: string;
|
||||
status: string;
|
||||
approval_id?: string;
|
||||
params?: Record<string, unknown>;
|
||||
created_at?: string;
|
||||
started_at?: string;
|
||||
finished_at?: string;
|
||||
result?: { error?: string; ok?: boolean; [key: string]: unknown };
|
||||
};
|
||||
|
||||
export type TrainingSummary = {
|
||||
total: number;
|
||||
running: number;
|
||||
queued: number;
|
||||
succeeded: number;
|
||||
failed: number;
|
||||
};
|
||||
|
||||
export type TrainingRecord = JobRecord & {
|
||||
action_label?: string;
|
||||
project?: string;
|
||||
kind?: string;
|
||||
task?: string | null;
|
||||
track?: string;
|
||||
weight_path?: string | null;
|
||||
metrics?: Record<string, unknown>;
|
||||
error?: string | null;
|
||||
duration_sec?: number | null;
|
||||
approval?: ApprovalRecord | null;
|
||||
};
|
||||
|
||||
export type ModelRegistry = {
|
||||
project: string;
|
||||
task?: string;
|
||||
version?: Record<string, unknown>;
|
||||
tasks?: Record<string, Record<string, unknown>>;
|
||||
eval_history?: Record<string, unknown>[];
|
||||
};
|
||||
|
||||
export type DataCandidate = {
|
||||
id: string;
|
||||
project: string;
|
||||
task?: string;
|
||||
status: string;
|
||||
source_type: string;
|
||||
original_name?: string;
|
||||
upload_path: string;
|
||||
analyzed_source_path?: string;
|
||||
format_id?: string;
|
||||
split_counts?: Record<string, number>;
|
||||
error_message?: string;
|
||||
upload_size_bytes?: number;
|
||||
submitted_by_name?: string;
|
||||
analysis_job_id?: string;
|
||||
created_at?: string;
|
||||
updated_at?: string;
|
||||
};
|
||||
|
||||
export type InspectUploadResponse = {
|
||||
ok: boolean;
|
||||
normalized: {
|
||||
format_id: string;
|
||||
project: string;
|
||||
task?: string;
|
||||
source_path: string;
|
||||
split_counts?: Record<string, number>;
|
||||
sample_count?: number;
|
||||
annotation_count?: number;
|
||||
artifacts?: string[];
|
||||
warnings?: string[];
|
||||
extra?: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
120
platform/web/src/auth/AuthContext.tsx
Normal file
120
platform/web/src/auth/AuthContext.tsx
Normal file
@@ -0,0 +1,120 @@
|
||||
import { createContext, useCallback, useContext, useEffect, useMemo, useState, type ReactNode } from "react";
|
||||
import { api, type AuthUser } from "../api/client";
|
||||
|
||||
const TOKEN_KEY = "as_access_token";
|
||||
|
||||
type AuthState = {
|
||||
user: AuthUser | null;
|
||||
loading: boolean;
|
||||
token: string | null;
|
||||
authConfig: { feishu_enabled: boolean; dev_auth_enabled: boolean } | null;
|
||||
loginFeishu: () => void;
|
||||
loginDev: (name?: string) => Promise<void>;
|
||||
logout: () => void;
|
||||
refreshUser: () => Promise<void>;
|
||||
hasPermission: (code: string) => boolean;
|
||||
};
|
||||
|
||||
const AuthCtx = createContext<AuthState | null>(null);
|
||||
|
||||
export function AuthProvider({ children }: { children: ReactNode }) {
|
||||
const [user, setUser] = useState<AuthUser | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [token, setToken] = useState<string | null>(() => localStorage.getItem(TOKEN_KEY));
|
||||
const [authConfig, setAuthConfig] = useState<{ feishu_enabled: boolean; dev_auth_enabled: boolean } | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
// 支持后端回调重定向到 /?token=...
|
||||
const url = new URL(window.location.href);
|
||||
const tokenFromQuery = url.searchParams.get("token");
|
||||
if (tokenFromQuery) {
|
||||
localStorage.setItem(TOKEN_KEY, tokenFromQuery);
|
||||
api.setToken(tokenFromQuery);
|
||||
url.searchParams.delete("token");
|
||||
window.history.replaceState({}, "", url.toString());
|
||||
setToken(tokenFromQuery);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const applyToken = useCallback((t: string | null) => {
|
||||
setToken(t);
|
||||
if (t) localStorage.setItem(TOKEN_KEY, t);
|
||||
else localStorage.removeItem(TOKEN_KEY);
|
||||
api.setToken(t);
|
||||
}, []);
|
||||
|
||||
const refreshUser = useCallback(async () => {
|
||||
if (!token) {
|
||||
setUser(null);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const me = await api.me();
|
||||
setUser(me);
|
||||
} catch {
|
||||
applyToken(null);
|
||||
setUser(null);
|
||||
}
|
||||
}, [token, applyToken]);
|
||||
|
||||
useEffect(() => {
|
||||
api.authConfig().then(setAuthConfig).catch(() => setAuthConfig({ feishu_enabled: false, dev_auth_enabled: true }));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
api.setToken(token);
|
||||
if (token) {
|
||||
refreshUser().finally(() => setLoading(false));
|
||||
} else {
|
||||
setUser(null);
|
||||
setLoading(false);
|
||||
}
|
||||
}, [token, refreshUser]);
|
||||
|
||||
const loginFeishu = () => {
|
||||
window.location.href = "/api/v1/auth/feishu/authorize";
|
||||
};
|
||||
|
||||
const loginDev = async (name?: string) => {
|
||||
const res = await api.devLogin(name);
|
||||
applyToken(res.access_token);
|
||||
setUser(res.user);
|
||||
};
|
||||
|
||||
const logout = () => {
|
||||
applyToken(null);
|
||||
setUser(null);
|
||||
};
|
||||
|
||||
const hasPermission = useCallback(
|
||||
(code: string) => {
|
||||
if (!user) return false;
|
||||
const perms = user.permissions || [];
|
||||
return perms.includes("*") || perms.includes(code);
|
||||
},
|
||||
[user]
|
||||
);
|
||||
|
||||
const value = useMemo(
|
||||
() => ({
|
||||
user,
|
||||
loading,
|
||||
token,
|
||||
loginFeishu,
|
||||
loginDev,
|
||||
logout,
|
||||
refreshUser,
|
||||
hasPermission,
|
||||
authConfig,
|
||||
}),
|
||||
[user, loading, token, refreshUser, hasPermission, authConfig]
|
||||
);
|
||||
|
||||
return <AuthCtx.Provider value={value}>{children}</AuthCtx.Provider>;
|
||||
}
|
||||
|
||||
export function useAuth() {
|
||||
const ctx = useContext(AuthCtx);
|
||||
if (!ctx) throw new Error("useAuth outside AuthProvider");
|
||||
return ctx;
|
||||
}
|
||||
18
platform/web/src/auth/RequireAuth.tsx
Normal file
18
platform/web/src/auth/RequireAuth.tsx
Normal file
@@ -0,0 +1,18 @@
|
||||
import { Navigate } from "react-router-dom";
|
||||
import { useAuth } from "../auth/AuthContext";
|
||||
import type { ReactNode } from "react";
|
||||
|
||||
export function RequireAuth({ children }: { children: ReactNode }) {
|
||||
const { user, loading } = useAuth();
|
||||
if (loading) return <p className="empty-state">登录验证中…</p>;
|
||||
if (!user) return <Navigate to="/login" replace />;
|
||||
return <>{children}</>;
|
||||
}
|
||||
|
||||
export function RequirePermission({ code, children }: { code: string; children: ReactNode }) {
|
||||
const { hasPermission } = useAuth();
|
||||
if (!hasPermission(code)) {
|
||||
return <p className="empty-state">无权访问此页面(需要 {code})</p>;
|
||||
}
|
||||
return <>{children}</>;
|
||||
}
|
||||
111
platform/web/src/components/Layout.tsx
Normal file
111
platform/web/src/components/Layout.tsx
Normal file
@@ -0,0 +1,111 @@
|
||||
import { NavLink, Outlet } from "react-router-dom";
|
||||
import { useEffect, useState } from "react";
|
||||
import { api } from "../api/client";
|
||||
import { useAuth } from "../auth/AuthContext";
|
||||
|
||||
const NAV = [
|
||||
{ to: "/labeling", icon: "◎", label: "送标工作台", badge: "pending" as const, perm: "read:pending" },
|
||||
{ to: "/catalog", icon: "▤", label: "数据目录", perm: "read:catalog" },
|
||||
{ to: "/audit", icon: "✓", label: "审核管理", badge: "audit" as const, perm: "read:audit" },
|
||||
{ to: "/jobs", icon: "◷", label: "任务监控", perm: "read:jobs" },
|
||||
{ to: "/training", icon: "▶", label: "模型训练", perm: "write:approval_submit" },
|
||||
{ to: "/logs", icon: "☰", label: "审计日志", perm: "read:audit" },
|
||||
];
|
||||
|
||||
export function Layout() {
|
||||
const { user, logout, hasPermission } = useAuth();
|
||||
const [apiOk, setApiOk] = useState<boolean | null>(null);
|
||||
const [pendingN, setPendingN] = useState(0);
|
||||
const [auditN, setAuditN] = useState(0);
|
||||
|
||||
const refreshMeta = async () => {
|
||||
try {
|
||||
await api.health();
|
||||
setApiOk(true);
|
||||
if (hasPermission("read:pending")) {
|
||||
const pending = await api.pending();
|
||||
const actionable = (pending.batches || []).filter((b) =>
|
||||
["returned", "raw_pool", "out_for_labeling"].includes(b.stage)
|
||||
);
|
||||
setPendingN(actionable.length);
|
||||
}
|
||||
if (hasPermission("read:audit")) {
|
||||
const aud = await api.listApprovals("pending");
|
||||
setAuditN(aud.items?.length || 0);
|
||||
}
|
||||
} catch {
|
||||
setApiOk(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
refreshMeta();
|
||||
const t = setInterval(refreshMeta, 30000);
|
||||
return () => clearInterval(t);
|
||||
}, [user]);
|
||||
|
||||
const path = location.pathname.replace(/^\//, "").split("/")[0] || "labeling";
|
||||
const title = NAV.find((n) => n.to.slice(1) === path)?.label || "送标工作台";
|
||||
const roleLabel = user?.roles?.map((r) => r.name).join(" · ") || "";
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<aside className="sidebar">
|
||||
<div className="brand">
|
||||
<div className="brand-logo" aria-hidden="true">
|
||||
<svg viewBox="0 0 40 40" fill="none">
|
||||
<rect width="40" height="40" rx="10" fill="url(#g)" />
|
||||
<path d="M12 28V12h6l4 10 4-10h6v16h-5V18l-4 10h-4l-4-10v10H12z" fill="white" fillOpacity="0.95" />
|
||||
<defs>
|
||||
<linearGradient id="g" x1="0" y1="0" x2="40" y2="40">
|
||||
<stop stopColor="#0ea5e9" />
|
||||
<stop offset="1" stopColor="#06b6d4" />
|
||||
</linearGradient>
|
||||
</defs>
|
||||
</svg>
|
||||
</div>
|
||||
<div className="brand-text">
|
||||
<span className="brand-company">Huaxu Sentinel</span>
|
||||
<span className="brand-product">HSAP · 主动安全算法平台</span>
|
||||
</div>
|
||||
</div>
|
||||
<nav className="nav">
|
||||
{NAV.filter((n) => hasPermission(n.perm)).map((n) => (
|
||||
<NavLink key={n.to} to={n.to} className={({ isActive }) => "nav-item" + (isActive ? " active" : "")}>
|
||||
<span className="nav-icon">{n.icon}</span> {n.label}
|
||||
{n.badge === "pending" && pendingN > 0 && <span className="nav-badge">{pendingN}</span>}
|
||||
{n.badge === "audit" && auditN > 0 && <span className="nav-badge">{auditN}</span>}
|
||||
</NavLink>
|
||||
))}
|
||||
</nav>
|
||||
<div className="sidebar-footer">
|
||||
<div className="user-chip">
|
||||
{user?.avatar_url ? <img src={user.avatar_url} alt="" className="user-avatar" /> : <span className="user-avatar user-avatar-ph">{user?.name?.[0]}</span>}
|
||||
<div>
|
||||
<div className="user-name">{user?.name}</div>
|
||||
<div className="user-role text-dim">{roleLabel}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="env-chip">
|
||||
<span className="env-dot" style={{ background: apiOk ? "var(--success)" : apiOk === false ? "var(--warning)" : undefined }} />
|
||||
<span>{apiOk ? "算法服务运行中" : apiOk === false ? "算法服务离线" : "服务检测中…"}</span>
|
||||
</div>
|
||||
<button type="button" className="btn btn-sm btn-ghost btn-logout" onClick={logout}>退出登录</button>
|
||||
</div>
|
||||
</aside>
|
||||
<div className="main-wrap">
|
||||
<header className="topbar">
|
||||
<div className="topbar-title">
|
||||
<h1>{title}</h1>
|
||||
</div>
|
||||
<div className="topbar-actions">
|
||||
<button type="button" className="btn btn-ghost" onClick={() => refreshMeta()}>刷新</button>
|
||||
</div>
|
||||
</header>
|
||||
<main className="content">
|
||||
<Outlet context={{ refreshMeta }} />
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
30
platform/web/src/components/Toast.tsx
Normal file
30
platform/web/src/components/Toast.tsx
Normal file
@@ -0,0 +1,30 @@
|
||||
import { createContext, useCallback, useContext, useState, type ReactNode } from "react";
|
||||
|
||||
type Toast = { id: number; msg: string; err?: boolean };
|
||||
|
||||
const ToastCtx = createContext<(msg: string, err?: boolean) => void>(() => {});
|
||||
|
||||
export function ToastProvider({ children }: { children: ReactNode }) {
|
||||
const [toasts, setToasts] = useState<Toast[]>([]);
|
||||
const toast = useCallback((msg: string, err?: boolean) => {
|
||||
const id = Date.now();
|
||||
setToasts((t) => [...t, { id, msg, err }]);
|
||||
setTimeout(() => setToasts((t) => t.filter((x) => x.id !== id)), 4000);
|
||||
}, []);
|
||||
return (
|
||||
<ToastCtx.Provider value={toast}>
|
||||
{children}
|
||||
<div className="toast-container">
|
||||
{toasts.map((t) => (
|
||||
<div key={t.id} className={"toast" + (t.err ? " toast-err" : "")}>
|
||||
{t.msg}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</ToastCtx.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
export function useToast() {
|
||||
return useContext(ToastCtx);
|
||||
}
|
||||
2
platform/web/src/config/featureFlags.ts
Normal file
2
platform/web/src/config/featureFlags.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
/** 车道线数据可视化(暂时关闭;恢复时改为 import.meta.env.VITE_LANE_DATA_VIZ === "1") */
|
||||
export const LANE_DATA_VIZ_ENABLED = false;
|
||||
21
platform/web/src/lib/labeling.ts
Normal file
21
platform/web/src/lib/labeling.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
export const LabelingStates: Record<
|
||||
string,
|
||||
{ label: string; badge: string; hint: string }
|
||||
> = {
|
||||
raw_pool: { label: "原图池", badge: "badge-pending", hint: "仅有图像,待送标或等待标注回传" },
|
||||
out_for_labeling: { label: "送标中", badge: "badge-training", hint: "已导出清单,等待标注方回传" },
|
||||
returned: { label: "回传待入库", badge: "badge-staged", hint: "数据已落盘,可执行 build / add" },
|
||||
ingested: { label: "已入库", badge: "badge-evaluated", hint: "已完成 ingest 或建包" },
|
||||
};
|
||||
|
||||
export function forStage(stage: string) {
|
||||
return LabelingStates[stage] ?? { label: stage, badge: "badge-idle", hint: "" };
|
||||
}
|
||||
|
||||
export const DropPaths = {
|
||||
dms_inbox: (ws: string, task: string, batch: string) =>
|
||||
`${ws}/datasets/dms/inbox/${task}/${batch}/`,
|
||||
dms_sources: (ws: string, pack: string, task: string, batch: string) =>
|
||||
`${ws}/datasets/dms/packs/${pack}/${task}/sources/${batch}/`,
|
||||
lane_add: (ws: string) => `${ws}/datasets/lane/ # archive 含 train_val_gt.txt`,
|
||||
};
|
||||
10
platform/web/src/main.tsx
Normal file
10
platform/web/src/main.tsx
Normal file
@@ -0,0 +1,10 @@
|
||||
import { StrictMode } from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import App from "./App";
|
||||
import "./styles/main.css";
|
||||
|
||||
createRoot(document.getElementById("root")!).render(
|
||||
<StrictMode>
|
||||
<App />
|
||||
</StrictMode>
|
||||
);
|
||||
367
platform/web/src/pages/Audit.tsx
Normal file
367
platform/web/src/pages/Audit.tsx
Normal file
@@ -0,0 +1,367 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { Link, useNavigate, useOutletContext, useParams } from "react-router-dom";
|
||||
import { api, type ApprovalRecord, type AuditImageItem, type AuditPreview } from "../api/client";
|
||||
import { useAuth } from "../auth/AuthContext";
|
||||
import { useToast } from "../components/Toast";
|
||||
|
||||
const STATUS: Record<string, { label: string; badge: string }> = {
|
||||
pending: { label: "待审核", badge: "badge-pending" },
|
||||
approved: { label: "已通过", badge: "badge-staged" },
|
||||
rejected: { label: "已驳回", badge: "badge-idle" },
|
||||
executed: { label: "已执行", badge: "badge-evaluated" },
|
||||
failed: { label: "执行失败", badge: "badge-pending" },
|
||||
running: { label: "执行中", badge: "badge-training" },
|
||||
};
|
||||
|
||||
type Ctx = { refreshMeta: () => void };
|
||||
|
||||
function AuthImage({
|
||||
approvalId,
|
||||
imageId,
|
||||
thumb = true,
|
||||
alt,
|
||||
className,
|
||||
onClick,
|
||||
}: {
|
||||
approvalId: string;
|
||||
imageId: string;
|
||||
thumb?: boolean;
|
||||
alt: string;
|
||||
className?: string;
|
||||
onClick?: () => void;
|
||||
}) {
|
||||
const [src, setSrc] = useState<string | null>(null);
|
||||
const [failed, setFailed] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
let revoked = false;
|
||||
let objectUrl: string | null = null;
|
||||
setFailed(false);
|
||||
setSrc(null);
|
||||
|
||||
api.fetchApprovalImageBlob(approvalId, imageId, thumb).then(
|
||||
(url) => {
|
||||
if (revoked) {
|
||||
URL.revokeObjectURL(url);
|
||||
return;
|
||||
}
|
||||
objectUrl = url;
|
||||
setSrc(url);
|
||||
},
|
||||
() => {
|
||||
if (!revoked) setFailed(true);
|
||||
}
|
||||
);
|
||||
|
||||
return () => {
|
||||
revoked = true;
|
||||
if (objectUrl) URL.revokeObjectURL(objectUrl);
|
||||
};
|
||||
}, [approvalId, imageId, thumb]);
|
||||
|
||||
if (failed) {
|
||||
return <div className={`audit-img-placeholder ${className || ""}`}>加载失败</div>;
|
||||
}
|
||||
if (!src) {
|
||||
return <div className={`audit-img-placeholder audit-img-loading ${className || ""}`}>…</div>;
|
||||
}
|
||||
return <img src={src} alt={alt} className={className} onClick={onClick} loading="lazy" />;
|
||||
}
|
||||
|
||||
export function AuditDetailPage() {
|
||||
const { id } = useParams<{ id: string }>();
|
||||
const { refreshMeta } = useOutletContext<Ctx>();
|
||||
const toast = useToast();
|
||||
const { hasPermission } = useAuth();
|
||||
const canReview = hasPermission("write:approval_review");
|
||||
|
||||
const [preview, setPreview] = useState<AuditPreview | null>(null);
|
||||
const [images, setImages] = useState<AuditImageItem[]>([]);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [loadingMore, setLoadingMore] = useState(false);
|
||||
const [lightbox, setLightbox] = useState<AuditImageItem | null>(null);
|
||||
const [batchFilter, setBatchFilter] = useState("");
|
||||
const sentinelRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const approvalId = id || "";
|
||||
|
||||
const loadPreview = useCallback(async () => {
|
||||
if (!approvalId) return;
|
||||
const data = await api.getApprovalPreview(approvalId);
|
||||
setPreview(data);
|
||||
}, [approvalId]);
|
||||
|
||||
const loadImages = useCallback(
|
||||
async (append = false) => {
|
||||
if (!approvalId) return;
|
||||
if (append) setLoadingMore(true);
|
||||
else setLoading(true);
|
||||
try {
|
||||
const offset = append ? images.length : 0;
|
||||
const data = await api.listApprovalImages(approvalId, offset, 60);
|
||||
setTotal(data.total);
|
||||
setImages((prev) => (append ? [...prev, ...data.items] : data.items));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
setLoadingMore(false);
|
||||
}
|
||||
},
|
||||
[approvalId, images.length]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
loadPreview().catch((e) => toast(String(e), true));
|
||||
loadImages(false).catch((e) => toast(String(e), true));
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [approvalId]);
|
||||
|
||||
useEffect(() => {
|
||||
const el = sentinelRef.current;
|
||||
if (!el || loading || loadingMore || images.length >= total) return;
|
||||
const obs = new IntersectionObserver(
|
||||
(entries) => {
|
||||
if (entries[0]?.isIntersecting && images.length < total) {
|
||||
loadImages(true).catch((e) => toast(String(e), true));
|
||||
}
|
||||
},
|
||||
{ rootMargin: "200px" }
|
||||
);
|
||||
obs.observe(el);
|
||||
return () => obs.disconnect();
|
||||
}, [images.length, total, loading, loadingMore, loadImages, toast]);
|
||||
|
||||
const rec: ApprovalRecord | undefined = preview?.approval;
|
||||
const st = rec ? STATUS[rec.status] || { label: rec.status, badge: "badge-idle" } : null;
|
||||
const batches = [...new Set(images.map((i) => i.batch))].sort();
|
||||
const shown = batchFilter ? images.filter((i) => i.batch === batchFilter) : images;
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="audit-detail-head panel panel-compact">
|
||||
<div className="panel-body">
|
||||
<div className="audit-detail-nav">
|
||||
<Link to="/audit" className="btn btn-sm btn-ghost">← 返回队列</Link>
|
||||
{rec && st && (
|
||||
<span className={`badge ${st.badge}`}>{st.label}</span>
|
||||
)}
|
||||
</div>
|
||||
{rec && (
|
||||
<>
|
||||
<h2 className="audit-detail-title">{rec.action_label || rec.action}</h2>
|
||||
<p className="text-dim audit-detail-meta">
|
||||
{rec.id} · {rec.submitted_by || "—"} · {rec.submitted_at?.slice(0, 19)}
|
||||
</p>
|
||||
{preview?.scope_label && (
|
||||
<p className="audit-scope-label">{preview.scope_label}</p>
|
||||
)}
|
||||
{preview?.batches?.length ? (
|
||||
<div className="audit-batch-tags">
|
||||
{preview.batches.map((b) => (
|
||||
<span key={`${b.location}:${b.batch}`} className={`audit-batch-tag ${b.exists ? "" : "missing"}`}>
|
||||
{b.location}/{b.batch}
|
||||
{!b.exists && " (目录不存在)"}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
{rec.status === "pending" && canReview && (
|
||||
<div className="audit-detail-actions">
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-primary"
|
||||
onClick={async () => {
|
||||
try {
|
||||
await api.approve(rec.id, "批准");
|
||||
toast("已批准");
|
||||
refreshMeta();
|
||||
loadPreview();
|
||||
} catch (e) {
|
||||
toast(String(e), true);
|
||||
}
|
||||
}}
|
||||
>
|
||||
批准执行
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-ghost"
|
||||
onClick={async () => {
|
||||
const reason = prompt("驳回原因");
|
||||
if (reason === null) return;
|
||||
await api.reject(rec.id, reason);
|
||||
toast("已驳回");
|
||||
refreshMeta();
|
||||
loadPreview();
|
||||
}}
|
||||
>
|
||||
驳回
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="panel">
|
||||
<div className="panel-header">
|
||||
<h2>送标注图 · GT 可视化</h2>
|
||||
<div className="audit-gallery-toolbar">
|
||||
<span className="text-dim">
|
||||
已加载 {images.length} / {total} 张
|
||||
</span>
|
||||
{batches.length > 1 && (
|
||||
<select
|
||||
className="audit-batch-select"
|
||||
value={batchFilter}
|
||||
onChange={(e) => setBatchFilter(e.target.value)}
|
||||
>
|
||||
<option value="">全部批次</option>
|
||||
{batches.map((b) => (
|
||||
<option key={b} value={b}>{b}</option>
|
||||
))}
|
||||
</select>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="panel-body">
|
||||
{loading && !images.length ? (
|
||||
<p className="empty-state">加载图像…</p>
|
||||
) : total === 0 ? (
|
||||
<p className="empty-state">该审核单范围内未找到送标注图像</p>
|
||||
) : (
|
||||
<>
|
||||
<div className="audit-gallery">
|
||||
{shown.map((item) => (
|
||||
<button
|
||||
key={item.id}
|
||||
type="button"
|
||||
className={`audit-gallery-item ${item.missing_label ? "no-label" : ""}`}
|
||||
onClick={() => setLightbox(item)}
|
||||
title={item.filename}
|
||||
>
|
||||
<AuthImage
|
||||
approvalId={approvalId}
|
||||
imageId={item.id}
|
||||
thumb
|
||||
alt={item.filename}
|
||||
className="audit-gallery-img"
|
||||
/>
|
||||
<div className="audit-gallery-cap">
|
||||
<span className="mono">{item.filename}</span>
|
||||
<span>
|
||||
{item.batch} · {item.split}
|
||||
{item.box_count > 0 ? ` · ${item.box_count} 框` : item.missing_label ? " · 无标注" : ""}
|
||||
</span>
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
{loadingMore && <p className="empty-state">加载更多…</p>}
|
||||
<div ref={sentinelRef} className="audit-sentinel" />
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{lightbox && (
|
||||
<div className="audit-lightbox" onClick={() => setLightbox(null)} role="presentation">
|
||||
<div className="audit-lightbox-inner" onClick={(e) => e.stopPropagation()}>
|
||||
<button type="button" className="audit-lightbox-close btn btn-sm btn-ghost" onClick={() => setLightbox(null)}>
|
||||
关闭
|
||||
</button>
|
||||
<AuthImage
|
||||
approvalId={approvalId}
|
||||
imageId={lightbox.id}
|
||||
thumb={false}
|
||||
alt={lightbox.filename}
|
||||
className="audit-lightbox-img"
|
||||
/>
|
||||
<div className="audit-lightbox-meta">
|
||||
<strong>{lightbox.filename}</strong>
|
||||
<span>{lightbox.batch} · {lightbox.split} · {lightbox.box_count} 个标注框</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export function AuditPage() {
|
||||
const navigate = useNavigate();
|
||||
const { refreshMeta } = useOutletContext<Ctx>();
|
||||
const toast = useToast();
|
||||
const { hasPermission } = useAuth();
|
||||
const canReview = hasPermission("write:approval_review");
|
||||
const [filter, setFilter] = useState("pending");
|
||||
const [items, setItems] = useState<ApprovalRecord[]>([]);
|
||||
|
||||
const load = async () => {
|
||||
const data = await api.listApprovals(filter || undefined);
|
||||
setItems(data.items || []);
|
||||
};
|
||||
|
||||
useEffect(() => { load(); }, [filter]);
|
||||
|
||||
return (
|
||||
<div className="panel">
|
||||
<div className="panel-header">
|
||||
<h2>审核队列</h2>
|
||||
<div className="audit-filters">
|
||||
{["pending", "executed", "rejected", "failed", ""].map((f) => (
|
||||
<button key={f || "all"} type="button" className={`btn btn-sm ${filter === f ? "btn-primary" : "btn-ghost"}`} onClick={() => setFilter(f)}>
|
||||
{f === "" ? "全部" : STATUS[f]?.label || f}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div className="panel-body table-wrap">
|
||||
<table className="data-table audit-table">
|
||||
<thead><tr><th>单号</th><th>动作</th><th>状态</th><th>提交人</th><th>时间</th><th>参数</th><th>操作</th></tr></thead>
|
||||
<tbody>
|
||||
{items.map((r) => {
|
||||
const st = STATUS[r.status] || { label: r.status, badge: "badge-idle" };
|
||||
return (
|
||||
<tr
|
||||
key={r.id}
|
||||
className="audit-row-clickable"
|
||||
onClick={() => navigate(`/audit/${r.id}`)}
|
||||
>
|
||||
<td className="mono text-sm">{r.id}</td>
|
||||
<td>{r.action_label || r.action}</td>
|
||||
<td><span className={`badge ${st.badge}`}>{st.label}</span></td>
|
||||
<td>{r.submitted_by || "—"}</td>
|
||||
<td className="text-sm">{r.submitted_at?.slice(0, 19)}</td>
|
||||
<td className="text-sm"><pre className="params-pre">{JSON.stringify(r.params || {})}</pre></td>
|
||||
<td onClick={(e) => e.stopPropagation()}>
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => navigate(`/audit/${r.id}`)}>
|
||||
查看标注图
|
||||
</button>
|
||||
{r.status === "pending" && canReview ? (
|
||||
<>
|
||||
<button type="button" className="btn btn-sm btn-primary" onClick={async () => {
|
||||
try { await api.approve(r.id, "批准"); toast("已批准"); load(); refreshMeta(); }
|
||||
catch (e) { toast(String(e), true); }
|
||||
}}>批准</button>
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={async () => {
|
||||
await api.reject(r.id, prompt("驳回原因") || "");
|
||||
load(); refreshMeta();
|
||||
}}>驳回</button>
|
||||
</>
|
||||
) : r.status === "pending" ? (
|
||||
<span className="text-dim">无审核权限</span>
|
||||
) : (
|
||||
r.review_comment || r.result?.error || ""
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
22
platform/web/src/pages/AuthCallback.tsx
Normal file
22
platform/web/src/pages/AuthCallback.tsx
Normal file
@@ -0,0 +1,22 @@
|
||||
import { useEffect } from "react";
|
||||
import { useNavigate, useSearchParams } from "react-router-dom";
|
||||
import { api } from "../api/client";
|
||||
|
||||
export function AuthCallbackPage() {
|
||||
const [params] = useSearchParams();
|
||||
const navigate = useNavigate();
|
||||
|
||||
useEffect(() => {
|
||||
const token = params.get("token");
|
||||
if (token) {
|
||||
localStorage.setItem("as_access_token", token);
|
||||
api.setToken(token);
|
||||
navigate("/labeling", { replace: true });
|
||||
window.location.reload();
|
||||
} else {
|
||||
navigate("/login", { replace: true });
|
||||
}
|
||||
}, [params, navigate]);
|
||||
|
||||
return <p className="empty-state">登录处理中…</p>;
|
||||
}
|
||||
651
platform/web/src/pages/Catalog.tsx
Normal file
651
platform/web/src/pages/Catalog.tsx
Normal file
@@ -0,0 +1,651 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { api, type CatalogReport, type InspectUploadResponse } from "../api/client";
|
||||
import { LANE_DATA_VIZ_ENABLED } from "../config/featureFlags";
|
||||
|
||||
type CatalogVersion = {
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
train_images?: number;
|
||||
val_images?: number;
|
||||
test_images?: number;
|
||||
class_counts?: Record<string, number>;
|
||||
path?: string;
|
||||
role?: string;
|
||||
frozen?: boolean;
|
||||
label_files?: number;
|
||||
total_boxes?: number;
|
||||
sampled?: boolean;
|
||||
bbox_points?: [number, number][];
|
||||
lane_quality?: {
|
||||
analyzed_frames?: number;
|
||||
lane_count_hist?: Record<string, number>;
|
||||
length_hist?: { left: number; right: number; count: number }[];
|
||||
curvature_hist?: { left: number; right: number; count: number }[];
|
||||
};
|
||||
};
|
||||
|
||||
function polarToCartesian(cx: number, cy: number, r: number, angleDeg: number) {
|
||||
const rad = (angleDeg - 90) * (Math.PI / 180);
|
||||
return { x: cx + r * Math.cos(rad), y: cy + r * Math.sin(rad) };
|
||||
}
|
||||
|
||||
function describeArc(cx: number, cy: number, r: number, startAngle: number, endAngle: number) {
|
||||
const start = polarToCartesian(cx, cy, r, endAngle);
|
||||
const end = polarToCartesian(cx, cy, r, startAngle);
|
||||
const largeArcFlag = endAngle - startAngle <= 180 ? "0" : "1";
|
||||
return `M ${start.x} ${start.y} A ${r} ${r} 0 ${largeArcFlag} 0 ${end.x} ${end.y}`;
|
||||
}
|
||||
|
||||
function radarPoints(values: number[], cx = 110, cy = 110, radius = 76): string {
|
||||
if (!values.length) return "";
|
||||
return values
|
||||
.map((v, i) => {
|
||||
const angle = (-90 + (360 / values.length) * i) * (Math.PI / 180);
|
||||
const r = Math.max(0, Math.min(1, v)) * radius;
|
||||
const x = cx + r * Math.cos(angle);
|
||||
const y = cy + r * Math.sin(angle);
|
||||
return `${x},${y}`;
|
||||
})
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
export function CatalogPage() {
|
||||
const [cat, setCat] = useState<CatalogReport | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [loadError, setLoadError] = useState("");
|
||||
const [cacheHint, setCacheHint] = useState("");
|
||||
const [project, setProject] = useState<"dms" | "lane">("dms");
|
||||
const [selectedTask, setSelectedTask] = useState<string>("");
|
||||
const [selectedVersion, setSelectedVersion] = useState<string>("");
|
||||
const [inspectProject, setInspectProject] = useState<"dms" | "lane">("dms");
|
||||
const [inspectTask, setInspectTask] = useState<string>("dam");
|
||||
const [inspectPath, setInspectPath] = useState<string>("");
|
||||
const [inspecting, setInspecting] = useState(false);
|
||||
const [inspectResult, setInspectResult] = useState<InspectUploadResponse["normalized"] | null>(null);
|
||||
const [inspectError, setInspectError] = useState<string>("");
|
||||
|
||||
const load = async (refresh = false) => {
|
||||
setLoading(true);
|
||||
setLoadError("");
|
||||
try {
|
||||
const data = await api.catalog(refresh);
|
||||
setCat(data);
|
||||
const meta = data._cache;
|
||||
if (meta?.cached) {
|
||||
const src = meta.build_source === "reports" ? "报表缓存" : "扫描缓存";
|
||||
setCacheHint(`${src} · ${Math.round(meta.cache_age_sec || 0)}s 前更新`);
|
||||
} else if (refresh) {
|
||||
setCacheHint("已强制刷新");
|
||||
} else {
|
||||
setCacheHint("已重新扫描");
|
||||
}
|
||||
} catch (e) {
|
||||
setLoadError(e instanceof Error ? e.message : "加载失败");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const inspectSource = async () => {
|
||||
const sourcePath = inspectPath.trim();
|
||||
if (!sourcePath) {
|
||||
setInspectError("请先输入目录路径");
|
||||
return;
|
||||
}
|
||||
setInspecting(true);
|
||||
setInspectError("");
|
||||
try {
|
||||
const res = await api.inspectUploadPath(
|
||||
inspectProject,
|
||||
sourcePath,
|
||||
inspectProject === "dms" ? inspectTask : undefined
|
||||
);
|
||||
setInspectResult(res.normalized);
|
||||
} catch (e) {
|
||||
setInspectResult(null);
|
||||
setInspectError(e instanceof Error ? e.message : "目录分析失败");
|
||||
} finally {
|
||||
setInspecting(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => { load(); }, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!cat) return;
|
||||
const dmsTasks = Object.keys(cat.dms || {});
|
||||
const lanePacks = Object.keys(cat.lane || {});
|
||||
if (project === "dms") {
|
||||
const first = selectedTask || dmsTasks[0] || "";
|
||||
setSelectedTask(first);
|
||||
const firstPack = (cat.dms?.[first]?.packs?.[0]?.name) || "";
|
||||
setSelectedVersion(firstPack);
|
||||
} else {
|
||||
const first = selectedTask || lanePacks[0] || "";
|
||||
setSelectedTask(first);
|
||||
setSelectedVersion(first);
|
||||
}
|
||||
}, [cat, project]);
|
||||
|
||||
const dmsVersions = useMemo<CatalogVersion[]>(() => {
|
||||
if (!cat || project !== "dms" || !selectedTask) return [];
|
||||
return cat.dms?.[selectedTask]?.packs || [];
|
||||
}, [cat, project, selectedTask]);
|
||||
|
||||
const laneVersions = useMemo<CatalogVersion[]>(() => {
|
||||
if (!cat || project !== "lane") return [];
|
||||
return Object.entries(cat.lane || {}).map(([name, info]) => ({
|
||||
name,
|
||||
enabled: Boolean(info.enabled),
|
||||
train_images: info.train_lines || 0,
|
||||
val_images: info.val_lines || 0,
|
||||
test_images: info.test_lines || 0,
|
||||
class_counts: {},
|
||||
path: info.path,
|
||||
lane_quality: info.quality,
|
||||
}));
|
||||
}, [cat, project]);
|
||||
|
||||
const versions: CatalogVersion[] = project === "dms" ? dmsVersions : laneVersions;
|
||||
const current = versions.find((v) => v.name === selectedVersion) || versions[0];
|
||||
const classPairs = Object.entries((current?.class_counts || {}) as Record<string, number>)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10);
|
||||
const classMax = classPairs[0]?.[1] || 1;
|
||||
const classTotal = classPairs.reduce((acc, [, v]) => acc + v, 0);
|
||||
|
||||
const versionFeatures = useMemo(() => {
|
||||
if (project !== "dms") return [] as Array<{
|
||||
name: string;
|
||||
samples: number;
|
||||
boxes: number;
|
||||
density: number;
|
||||
avg_area: number;
|
||||
small_ratio: number;
|
||||
bbox_points: [number, number][];
|
||||
}>;
|
||||
return dmsVersions.map((v) => {
|
||||
const samples = (v.train_images || 0) + (v.val_images || 0) + (v.test_images || 0);
|
||||
const boxes = v.total_boxes || 0;
|
||||
const points = (v.bbox_points || []).filter((p) => p.length >= 2) as [number, number][];
|
||||
const areas = points.map(([w, h]) => w * h);
|
||||
const avg_area = areas.length ? areas.reduce((a, b) => a + b, 0) / areas.length : 0;
|
||||
const small_ratio = areas.length ? areas.filter((a) => a < 0.02).length / areas.length : 0;
|
||||
return {
|
||||
name: v.name,
|
||||
samples,
|
||||
boxes,
|
||||
density: samples > 0 ? boxes / samples : 0,
|
||||
avg_area,
|
||||
small_ratio,
|
||||
bbox_points: points,
|
||||
};
|
||||
});
|
||||
}, [project, dmsVersions]);
|
||||
|
||||
const featureMax = useMemo(() => {
|
||||
const samples = Math.max(1, ...versionFeatures.map((v) => v.samples));
|
||||
const density = Math.max(1, ...versionFeatures.map((v) => v.density));
|
||||
const boxes = Math.max(1, ...versionFeatures.map((v) => v.boxes));
|
||||
const avgArea = Math.max(1e-6, ...versionFeatures.map((v) => v.avg_area));
|
||||
const smallRatio = Math.max(1e-6, ...versionFeatures.map((v) => v.small_ratio));
|
||||
return { samples, density, boxes, avgArea, smallRatio };
|
||||
}, [versionFeatures]);
|
||||
const colorPalette = ["#22d3ee", "#a78bfa", "#34d399", "#f59e0b", "#60a5fa", "#f472b6", "#fb7185", "#94a3b8"];
|
||||
const currentFeature = versionFeatures.find((v) => v.name === current?.name);
|
||||
const whBins = useMemo(() => {
|
||||
const bins = Array.from({ length: 10 }, () => Array.from({ length: 10 }, () => 0));
|
||||
const points = currentFeature?.bbox_points || [];
|
||||
for (const [w, h] of points) {
|
||||
const xi = Math.min(9, Math.max(0, Math.floor(w * 10)));
|
||||
const yi = Math.min(9, Math.max(0, Math.floor(h * 10)));
|
||||
bins[9 - yi][xi] += 1;
|
||||
}
|
||||
let max = 0;
|
||||
for (const row of bins) for (const v of row) max = Math.max(max, v);
|
||||
return { bins, max };
|
||||
}, [currentFeature]);
|
||||
const radarAxes = ["样本量", "标签框", "框密度", "平均面积", "小目标占比"];
|
||||
const radarMetrics = useMemo(() => {
|
||||
return versionFeatures.slice(0, 4).map((v) => ({
|
||||
name: v.name,
|
||||
values: [
|
||||
v.samples / featureMax.samples,
|
||||
v.boxes / featureMax.boxes,
|
||||
v.density / featureMax.density,
|
||||
v.avg_area / featureMax.avgArea,
|
||||
v.small_ratio / featureMax.smallRatio,
|
||||
],
|
||||
}));
|
||||
}, [versionFeatures, featureMax]);
|
||||
const laneCountPairs = useMemo(() => {
|
||||
const hist = current?.lane_quality?.lane_count_hist || {};
|
||||
const entries = Object.entries(hist);
|
||||
entries.sort((a, b) => {
|
||||
const av = a[0] === "8+" ? 999 : Number(a[0]);
|
||||
const bv = b[0] === "8+" ? 999 : Number(b[0]);
|
||||
return av - bv;
|
||||
});
|
||||
return entries;
|
||||
}, [current]);
|
||||
const laneLengthHist = current?.lane_quality?.length_hist || [];
|
||||
const laneCurvHist = current?.lane_quality?.curvature_hist || [];
|
||||
const laneLengthMax = Math.max(1, ...laneLengthHist.map((x) => x.count));
|
||||
const laneCurvMax = Math.max(1, ...laneCurvHist.map((x) => x.count));
|
||||
|
||||
const kpis = useMemo(() => {
|
||||
const allDms = Object.values(cat?.dms || {}) as NonNullable<CatalogReport["dms"]>[string][];
|
||||
const allDmsPacks = allDms.flatMap((x) => x.packs || []);
|
||||
const allLane = Object.values(cat?.lane || {}) as NonNullable<CatalogReport["lane"]>[string][];
|
||||
const totalVersions = allDmsPacks.length + allLane.length;
|
||||
const enabledVersions = allDmsPacks.filter((x) => x.enabled).length + allLane.filter((x) => x.enabled).length;
|
||||
const totalSamples = allDmsPacks.reduce((acc, x) => acc + (x.train_images || 0) + (x.val_images || 0) + (x.test_images || 0), 0)
|
||||
+ allLane.reduce((acc, x) => acc + (x.train_lines || 0) + (x.val_lines || 0) + (x.test_lines || 0), 0);
|
||||
const totalBoxes = allDmsPacks.reduce((acc, x) => acc + (x.total_boxes || 0), 0);
|
||||
return { totalVersions, enabledVersions, totalSamples, totalBoxes };
|
||||
}, [cat]);
|
||||
const inspectSplits = useMemo(() => {
|
||||
const raw = inspectResult?.split_counts || {};
|
||||
return [
|
||||
["train", raw.train || 0],
|
||||
["val", raw.val || 0],
|
||||
["test", raw.test || 0],
|
||||
] as Array<[string, number]>;
|
||||
}, [inspectResult]);
|
||||
const inspectTotal = inspectSplits.reduce((acc, [, v]) => acc + v, 0);
|
||||
|
||||
const fmt = (n: number) => n.toLocaleString("zh-CN");
|
||||
|
||||
if (loading && !cat) return <p className="empty-state">加载 catalog 数据中…</p>;
|
||||
if (loadError && !cat) return <p className="empty-state">{loadError}</p>;
|
||||
if (!cat) return <p className="empty-state">暂无数据</p>;
|
||||
|
||||
return (
|
||||
<>
|
||||
{cacheHint && <p className="audit-note" style={{ marginBottom: 8 }}>{cacheHint}</p>}
|
||||
<div className="kpi-strip">
|
||||
<div className="kpi"><span className="kpi-val">{fmt(kpis.totalVersions)}</span><span className="kpi-lbl">接入数据版本数</span></div>
|
||||
<div className="kpi"><span className="kpi-val">{fmt(kpis.enabledVersions)}</span><span className="kpi-lbl">启用训练版本数</span></div>
|
||||
<div className="kpi"><span className="kpi-val">{fmt(kpis.totalSamples)}</span><span className="kpi-lbl">可用样本总量</span></div>
|
||||
<div className="kpi"><span className="kpi-val">{fmt(kpis.totalBoxes)}</span><span className="kpi-lbl">DMS 标签框总量</span></div>
|
||||
</div>
|
||||
<div className="panel">
|
||||
<div className="panel-header"><h2>现有目录快速分析</h2></div>
|
||||
<div className="panel-body">
|
||||
<div className="crud-form">
|
||||
<label className="field">
|
||||
<span>Project</span>
|
||||
<select value={inspectProject} onChange={(e) => setInspectProject(e.target.value as "dms" | "lane")}>
|
||||
<option value="dms">dms</option>
|
||||
<option value="lane">lane</option>
|
||||
</select>
|
||||
</label>
|
||||
{inspectProject === "dms" && (
|
||||
<label className="field">
|
||||
<span>Task</span>
|
||||
<input value={inspectTask} onChange={(e) => setInspectTask(e.target.value)} placeholder="如 dam" />
|
||||
</label>
|
||||
)}
|
||||
<label className="field" style={{ gridColumn: "1 / -1" }}>
|
||||
<span>目录路径</span>
|
||||
<input
|
||||
value={inspectPath}
|
||||
onChange={(e) => setInspectPath(e.target.value)}
|
||||
placeholder="例如 /data/workspace/DMS/inbox/dam/batch_xxx"
|
||||
/>
|
||||
</label>
|
||||
<div className="crud-actions" style={{ gridColumn: "1 / -1" }}>
|
||||
<button type="button" className="btn btn-primary" onClick={() => inspectSource()} disabled={inspecting}>
|
||||
{inspecting ? "分析中..." : "分析目录"}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{inspectError && <p className="empty-state">{inspectError}</p>}
|
||||
{inspectResult && (
|
||||
<div className="feature-grid" style={{ marginTop: 12 }}>
|
||||
<div className="feature-card">
|
||||
<h4>Split 分布环形图</h4>
|
||||
<div className="donut-wrap">
|
||||
<svg viewBox="0 0 200 200" className="donut-svg" role="img" aria-label="目录 split 分布">
|
||||
<circle cx="100" cy="100" r="70" fill="none" stroke="rgba(148,163,184,0.18)" strokeWidth="28" />
|
||||
{(() => {
|
||||
let angle = 0;
|
||||
return inspectSplits.map(([name, value], idx) => {
|
||||
const sweep = inspectTotal > 0 ? (value / inspectTotal) * 360 : 0;
|
||||
const d = describeArc(100, 100, 70, angle, angle + sweep);
|
||||
angle += sweep;
|
||||
return <path key={name} d={d} fill="none" stroke={colorPalette[idx % colorPalette.length]} strokeWidth="28" />;
|
||||
});
|
||||
})()}
|
||||
<text x="100" y="96" textAnchor="middle" className="donut-center-title">{inspectResult.format_id}</text>
|
||||
<text x="100" y="116" textAnchor="middle" className="donut-center-sub">{fmt(inspectTotal)} samples</text>
|
||||
</svg>
|
||||
<div className="donut-legend">
|
||||
{inspectSplits.map(([name, value], idx) => (
|
||||
<div key={`inspect-${name}`} className="legend-item">
|
||||
<i style={{ background: colorPalette[idx % colorPalette.length] }} />
|
||||
<span>{name}</span>
|
||||
<strong>{fmt(value)}</strong>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="feature-card">
|
||||
<h4>识别摘要</h4>
|
||||
<p className="audit-note">source: <code>{inspectResult.source_path}</code></p>
|
||||
<p className="audit-note">annotations: {fmt(inspectResult.annotation_count || 0)}</p>
|
||||
<p className="audit-note">artifacts: {(inspectResult.artifacts || []).join(", ") || "—"}</p>
|
||||
{(inspectResult.warnings || []).length > 0 ? (
|
||||
<ul className="audit-note">
|
||||
{(inspectResult.warnings || []).map((w, idx) => <li key={`warn-${idx}`}>{w}</li>)}
|
||||
</ul>
|
||||
) : (
|
||||
<p className="audit-note">warnings: none</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="grid-2-equal">
|
||||
<div className="panel">
|
||||
<div className="panel-header">
|
||||
<h2>数据版本列表</h2>
|
||||
<div className="audit-filters">
|
||||
<button type="button" className={`btn btn-sm ${project === "dms" ? "btn-primary" : "btn-ghost"}`} onClick={() => setProject("dms")}>DMS</button>
|
||||
<button type="button" className={`btn btn-sm ${project === "lane" ? "btn-primary" : "btn-ghost"}`} onClick={() => setProject("lane")}>Lane</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="panel-body">
|
||||
<div className="catalog-toolbar">
|
||||
<label className="field">
|
||||
<span>{project === "dms" ? "任务" : "包"}</span>
|
||||
<select
|
||||
value={selectedTask}
|
||||
onChange={(e) => {
|
||||
const value = e.target.value;
|
||||
setSelectedTask(value);
|
||||
const firstPack = project === "dms" ? (cat.dms?.[value]?.packs?.[0]?.name || "") : value;
|
||||
setSelectedVersion(firstPack);
|
||||
}}
|
||||
>
|
||||
{(project === "dms" ? Object.keys(cat.dms || {}) : Object.keys(cat.lane || {})).map((name) => (
|
||||
<option key={name} value={name}>{name}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
<div className="version-table">
|
||||
<table className="data-table compact">
|
||||
<thead><tr><th>版本</th><th>状态</th><th>Train</th><th>Val</th><th>Test</th><th>总量</th></tr></thead>
|
||||
<tbody>
|
||||
{versions.map((v) => {
|
||||
const total = (v.train_images || 0) + (v.val_images || 0) + (v.test_images || 0);
|
||||
return (
|
||||
<tr
|
||||
key={v.name}
|
||||
className={current?.name === v.name ? "row-active" : ""}
|
||||
onClick={() => setSelectedVersion(v.name)}
|
||||
style={{ cursor: "pointer" }}
|
||||
>
|
||||
<td><strong>{v.name}</strong></td>
|
||||
<td><span className={`badge ${v.enabled ? "badge-promoted" : "badge-idle"}`}>{v.enabled ? "启用中" : "未启用"}</span></td>
|
||||
<td>{fmt(v.train_images || 0)}</td>
|
||||
<td>{fmt(v.val_images || 0)}</td>
|
||||
<td>{fmt(v.test_images || 0)}</td>
|
||||
<td>{fmt(total)}</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="panel">
|
||||
<div className="panel-header"><h2>版本标签分布</h2></div>
|
||||
<div className="panel-body">
|
||||
{current ? (
|
||||
<div>
|
||||
<p className="catalog-intro">
|
||||
当前版本:<strong>{current.name}</strong> ·
|
||||
Train {fmt(current.train_images || 0)} / Val {fmt(current.val_images || 0)} / Test {fmt(current.test_images || 0)}
|
||||
</p>
|
||||
{classPairs.length > 0 ? (
|
||||
<div className="dist-list">
|
||||
{classPairs.map(([name, value]) => (
|
||||
<div key={name} className="dist-row">
|
||||
<div className="dist-meta"><span>{name}</span><strong>{value}</strong></div>
|
||||
<div className="progress-bar"><div className="progress-fill" style={{ width: `${Math.max(6, (value / classMax) * 100)}%` }} /></div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<p className="empty-state">当前版本暂无标签分布数据(Lane 或未入库标签)。</p>
|
||||
)}
|
||||
</div>
|
||||
) : <p className="empty-state">暂无可展示版本</p>}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="panel">
|
||||
<div className="panel-header"><h2>标签特征可视化</h2></div>
|
||||
<div className="panel-body">
|
||||
{project === "dms" ? (
|
||||
<>
|
||||
<div className="feature-grid">
|
||||
<div className="feature-card">
|
||||
<h4>类别结构环形图(当前版本)</h4>
|
||||
{classPairs.length > 0 ? (
|
||||
<div className="donut-wrap">
|
||||
<svg viewBox="0 0 200 200" className="donut-svg" role="img" aria-label="类别结构环形图">
|
||||
<circle cx="100" cy="100" r="70" fill="none" stroke="rgba(148,163,184,0.18)" strokeWidth="28" />
|
||||
{(() => {
|
||||
let angle = 0;
|
||||
return classPairs.map(([name, value], idx) => {
|
||||
const sweep = classTotal > 0 ? (value / classTotal) * 360 : 0;
|
||||
const d = describeArc(100, 100, 70, angle, angle + sweep);
|
||||
angle += sweep;
|
||||
return <path key={name} d={d} fill="none" stroke={colorPalette[idx % colorPalette.length]} strokeWidth="28" strokeLinecap="butt" />;
|
||||
});
|
||||
})()}
|
||||
<text x="100" y="96" textAnchor="middle" className="donut-center-title">{current?.name || "-"}</text>
|
||||
<text x="100" y="116" textAnchor="middle" className="donut-center-sub">{fmt(classTotal)} boxes</text>
|
||||
</svg>
|
||||
<div className="donut-legend">
|
||||
{classPairs.slice(0, 6).map(([name, value], idx) => (
|
||||
<div key={`legend-${name}`} className="legend-item">
|
||||
<i style={{ background: colorPalette[idx % colorPalette.length] }} />
|
||||
<span>{name}</span>
|
||||
<strong>{((value / Math.max(1, classTotal)) * 100).toFixed(1)}%</strong>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : <p className="empty-state">暂无类别数据</p>}
|
||||
</div>
|
||||
<div className="feature-card">
|
||||
<h4>目标宽高散点密度分布(当前版本)</h4>
|
||||
<svg viewBox="0 0 320 220" className="scatter-svg" role="img" aria-label="目标宽高散点密度分布">
|
||||
<rect x="40" y="20" width="250" height="170" fill="rgba(148,163,184,0.05)" stroke="rgba(148,163,184,0.25)" />
|
||||
{whBins.bins.map((row, yi) =>
|
||||
row.map((v, xi) => {
|
||||
if (!v) return null;
|
||||
const alpha = Math.min(0.85, v / Math.max(1, whBins.max));
|
||||
return (
|
||||
<rect
|
||||
key={`bin-${xi}-${yi}`}
|
||||
x={40 + xi * 25}
|
||||
y={20 + yi * 17}
|
||||
width={25}
|
||||
height={17}
|
||||
fill={`rgba(34,211,238,${alpha.toFixed(3)})`}
|
||||
/>
|
||||
);
|
||||
})
|
||||
)}
|
||||
{(currentFeature?.bbox_points || []).slice(0, 400).map(([w, h], idx) => (
|
||||
<circle
|
||||
key={`wh-${idx}`}
|
||||
cx={40 + w * 250}
|
||||
cy={190 - h * 170}
|
||||
r={1.6}
|
||||
fill="rgba(255,255,255,0.35)"
|
||||
/>
|
||||
))}
|
||||
<text x="294" y="205" textAnchor="end" className="scatter-axis">宽度 w</text>
|
||||
<text x="12" y="20" className="scatter-axis">高度 h</text>
|
||||
</svg>
|
||||
</div>
|
||||
<div className="feature-card">
|
||||
<h4>版本多指标雷达对比</h4>
|
||||
<div className="radar-wrap">
|
||||
<svg viewBox="0 0 220 220" className="radar-svg" role="img" aria-label="版本多指标雷达图">
|
||||
{[0.25, 0.5, 0.75, 1].map((lv) => (
|
||||
<polygon
|
||||
key={`grid-${lv}`}
|
||||
points={radarPoints([lv, lv, lv, lv, lv], 110, 110, 76)}
|
||||
fill="none"
|
||||
stroke="rgba(148,163,184,0.2)"
|
||||
strokeWidth="1"
|
||||
/>
|
||||
))}
|
||||
{radarAxes.map((label, i) => {
|
||||
const pt = radarPoints([1, 1, 1, 1, 1], 110, 110, 90).split(" ")[i];
|
||||
const [x, y] = pt.split(",");
|
||||
return <text key={label} x={Number(x)} y={Number(y)} className="radar-axis">{label}</text>;
|
||||
})}
|
||||
{radarMetrics.map((m, idx) => (
|
||||
<polygon
|
||||
key={`radar-${m.name}`}
|
||||
points={radarPoints(m.values, 110, 110, 76)}
|
||||
fill={colorPalette[idx % colorPalette.length]}
|
||||
fillOpacity="0.16"
|
||||
stroke={colorPalette[idx % colorPalette.length]}
|
||||
strokeWidth="2"
|
||||
/>
|
||||
))}
|
||||
</svg>
|
||||
<div className="donut-legend">
|
||||
{radarMetrics.map((m, idx) => (
|
||||
<div key={`radar-leg-${m.name}`} className="legend-item">
|
||||
<i style={{ background: colorPalette[idx % colorPalette.length] }} />
|
||||
<span>{m.name}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="version-table" style={{ marginTop: 14 }}>
|
||||
<table className="data-table compact">
|
||||
<thead><tr><th>版本</th><th>Train</th><th>Val</th><th>Test</th><th>标签框</th><th>框/样本</th><th>均值面积</th></tr></thead>
|
||||
<tbody>
|
||||
{versionFeatures.map((v) => {
|
||||
const raw = dmsVersions.find((x) => x.name === v.name);
|
||||
return (
|
||||
<tr key={v.name}>
|
||||
<td>{v.name}</td>
|
||||
<td>{fmt(raw?.train_images || 0)}</td>
|
||||
<td>{fmt(raw?.val_images || 0)}</td>
|
||||
<td>{fmt(raw?.test_images || 0)}</td>
|
||||
<td>{fmt(v.boxes)}</td>
|
||||
<td>{v.density.toFixed(2)}</td>
|
||||
<td>{(v.avg_area * 100).toFixed(2)}%</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</>
|
||||
) : LANE_DATA_VIZ_ENABLED ? (
|
||||
<>
|
||||
<div className="feature-grid">
|
||||
<div className="feature-card">
|
||||
<h4>每帧车道线数量分布</h4>
|
||||
<div className="donut-wrap">
|
||||
<svg viewBox="0 0 200 200" className="donut-svg" role="img" aria-label="每帧车道线数量分布">
|
||||
<circle cx="100" cy="100" r="70" fill="none" stroke="rgba(148,163,184,0.18)" strokeWidth="28" />
|
||||
{(() => {
|
||||
const total = laneCountPairs.reduce((a, [, c]) => a + c, 0);
|
||||
let angle = 0;
|
||||
return laneCountPairs.map(([bucket, cnt], idx) => {
|
||||
const sweep = total > 0 ? (cnt / total) * 360 : 0;
|
||||
const d = describeArc(100, 100, 70, angle, angle + sweep);
|
||||
angle += sweep;
|
||||
return <path key={bucket} d={d} fill="none" stroke={colorPalette[idx % colorPalette.length]} strokeWidth="28" />;
|
||||
});
|
||||
})()}
|
||||
<text x="100" y="100" textAnchor="middle" className="donut-center-title">线数分布</text>
|
||||
</svg>
|
||||
<div className="donut-legend">
|
||||
{laneCountPairs.map(([bucket, cnt], idx) => (
|
||||
<div key={`lc-${bucket}`} className="legend-item">
|
||||
<i style={{ background: colorPalette[idx % colorPalette.length] }} />
|
||||
<span>{bucket} 条</span>
|
||||
<strong>{fmt(cnt)}</strong>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="feature-card">
|
||||
<h4>线长度分布曲线</h4>
|
||||
<svg viewBox="0 0 320 200" className="scatter-svg" role="img" aria-label="线长度分布曲线">
|
||||
<line x1="36" y1="170" x2="300" y2="170" stroke="rgba(148,163,184,0.35)" />
|
||||
<line x1="36" y1="20" x2="36" y2="170" stroke="rgba(148,163,184,0.35)" />
|
||||
{laneLengthHist.length > 0 && (
|
||||
<polyline
|
||||
fill="none"
|
||||
stroke="#22d3ee"
|
||||
strokeWidth="2"
|
||||
points={laneLengthHist.map((b, i) => {
|
||||
const x = 36 + (i / Math.max(1, laneLengthHist.length - 1)) * 264;
|
||||
const y = 170 - (b.count / laneLengthMax) * 140;
|
||||
return `${x},${y}`;
|
||||
}).join(" ")}
|
||||
/>
|
||||
)}
|
||||
<text x="300" y="188" textAnchor="end" className="scatter-axis">长度区间</text>
|
||||
<text x="8" y="20" className="scatter-axis">频次</text>
|
||||
</svg>
|
||||
</div>
|
||||
<div className="feature-card">
|
||||
<h4>曲率分布曲线</h4>
|
||||
<svg viewBox="0 0 320 200" className="scatter-svg" role="img" aria-label="曲率分布曲线">
|
||||
<line x1="36" y1="170" x2="300" y2="170" stroke="rgba(148,163,184,0.35)" />
|
||||
<line x1="36" y1="20" x2="36" y2="170" stroke="rgba(148,163,184,0.35)" />
|
||||
{laneCurvHist.length > 0 && (
|
||||
<polyline
|
||||
fill="none"
|
||||
stroke="#a78bfa"
|
||||
strokeWidth="2"
|
||||
points={laneCurvHist.map((b, i) => {
|
||||
const x = 36 + (i / Math.max(1, laneCurvHist.length - 1)) * 264;
|
||||
const y = 170 - (b.count / laneCurvMax) * 140;
|
||||
return `${x},${y}`;
|
||||
}).join(" ")}
|
||||
/>
|
||||
)}
|
||||
<text x="300" y="188" textAnchor="end" className="scatter-axis">曲率区间</text>
|
||||
<text x="8" y="20" className="scatter-axis">频次</text>
|
||||
</svg>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<p className="empty-state">车道线数据可视化暂未开放(训练/评估不受影响)。</p>
|
||||
)}
|
||||
<div className="crud-actions" style={{ marginTop: 14 }}>
|
||||
<button type="button" className="btn btn-ghost" onClick={() => load(true)} disabled={loading}>
|
||||
{loading ? "刷新中..." : "刷新数据"}
|
||||
</button>
|
||||
<button type="button" className="btn btn-ghost" onClick={() => setSelectedVersion(versions[0]?.name || "")}>重置视图</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
262
platform/web/src/pages/Iterate.tsx
Normal file
262
platform/web/src/pages/Iterate.tsx
Normal file
@@ -0,0 +1,262 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { useOutletContext } from "react-router-dom";
|
||||
import { api, type CatalogReport, type JobRecord, type PendingReport } from "../api/client";
|
||||
import { useToast } from "../components/Toast";
|
||||
import { LANE_DATA_VIZ_ENABLED } from "../config/featureFlags";
|
||||
|
||||
type Ctx = { refreshMeta: () => void };
|
||||
type IterAction = "train_dms" | "train_lane" | "eval_dms" | "eval_lane" | "visualize_dms" | "visualize_lane";
|
||||
|
||||
const ITERATE_ACTIONS = new Set<IterAction>([
|
||||
"train_dms",
|
||||
"train_lane",
|
||||
"eval_dms",
|
||||
"eval_lane",
|
||||
"visualize_dms",
|
||||
"visualize_lane",
|
||||
]);
|
||||
|
||||
const ACTION_META: Record<IterAction, { label: string; project: "DMS" | "Lane"; kind: "训练" | "评估" | "可视化" }> = {
|
||||
train_dms: { label: "DMS 训练", project: "DMS", kind: "训练" },
|
||||
train_lane: { label: "Lane 训练", project: "Lane", kind: "训练" },
|
||||
eval_dms: { label: "DMS 评估", project: "DMS", kind: "评估" },
|
||||
eval_lane: { label: "Lane 评估", project: "Lane", kind: "评估" },
|
||||
visualize_dms: { label: "DMS 可视化", project: "DMS", kind: "可视化" },
|
||||
visualize_lane: { label: "Lane 可视化", project: "Lane", kind: "可视化" },
|
||||
};
|
||||
|
||||
function statusBadgeClass(status: string): string {
|
||||
if (status === "succeeded") return "badge-promoted";
|
||||
if (status === "running") return "badge-training";
|
||||
if (status === "failed") return "badge-pending";
|
||||
return "badge-idle";
|
||||
}
|
||||
|
||||
function fmtTime(ts?: string): string {
|
||||
if (!ts) return "—";
|
||||
const d = new Date(ts);
|
||||
if (Number.isNaN(d.getTime())) return ts;
|
||||
return d.toLocaleString("zh-CN", { hour12: false });
|
||||
}
|
||||
|
||||
function extractWeight(job: JobRecord): string | null {
|
||||
const res = (job.result || {}) as Record<string, unknown>;
|
||||
const params = (job.params || {}) as Record<string, unknown>;
|
||||
const candidates = [res.best_weights, res.candidate, res.model_path, res.weights, params.weights, params.model_path];
|
||||
for (const v of candidates) {
|
||||
if (typeof v === "string" && v.trim()) return v;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function IteratePage() {
|
||||
const { refreshMeta } = useOutletContext<Ctx>();
|
||||
const toast = useToast();
|
||||
const [pending, setPending] = useState<PendingReport | null>(null);
|
||||
const [cat, setCat] = useState<CatalogReport | null>(null);
|
||||
const [jobs, setJobs] = useState<JobRecord[]>([]);
|
||||
const [dmsTask, setDmsTask] = useState("dam");
|
||||
const [dmsTrack, setDmsTrack] = useState<"platform" | "local">("platform");
|
||||
const [dmsWeights, setDmsWeights] = useState("");
|
||||
const [laneTrack, setLaneTrack] = useState<"platform" | "local">("platform");
|
||||
const [laneModelPath, setLaneModelPath] = useState("");
|
||||
const [laneDataRoot, setLaneDataRoot] = useState("");
|
||||
const [laneTestList, setLaneTestList] = useState("list/test_gt.txt");
|
||||
const [selectedJobId, setSelectedJobId] = useState<string | null>(null);
|
||||
|
||||
const loadAll = async () => {
|
||||
const [p, c, j] = await Promise.all([api.pending(), api.catalog(), api.listJobs()]);
|
||||
setPending(p);
|
||||
setCat(c);
|
||||
setJobs(j.items || []);
|
||||
};
|
||||
|
||||
useEffect(() => { loadAll(); }, []);
|
||||
|
||||
useEffect(() => {
|
||||
const tasks = Object.keys(pending?.projects?.dms?.task_defs || {});
|
||||
if (tasks.length > 0 && !tasks.includes(dmsTask)) {
|
||||
setDmsTask(tasks[0]);
|
||||
}
|
||||
}, [pending]);
|
||||
|
||||
const submit = async (action: string, params: Record<string, unknown>) => {
|
||||
try {
|
||||
await api.submitApproval(action, params);
|
||||
toast(`已提交 ${action}`);
|
||||
refreshMeta();
|
||||
await loadAll();
|
||||
} catch (e) {
|
||||
toast(String(e), true);
|
||||
}
|
||||
};
|
||||
|
||||
const packRows = [
|
||||
...(pending?.projects?.dms?.active_packs || []).map((n) => ({ project: "dms", name: n, enabled: true })),
|
||||
...(pending?.projects?.dms?.not_enabled || []).map((n) => ({ project: "dms", name: n, enabled: false })),
|
||||
...Object.entries(cat?.lane || {}).map(([n, info]) => ({ project: "lane", name: n, enabled: info.enabled })),
|
||||
];
|
||||
|
||||
const iterJobs = useMemo(() => {
|
||||
return jobs.filter((j) => ITERATE_ACTIONS.has(j.action as IterAction)).slice(0, 30);
|
||||
}, [jobs]);
|
||||
const milestones = useMemo(() => [...iterJobs].reverse(), [iterJobs]);
|
||||
const selectedJob = useMemo(
|
||||
() => milestones.find((j) => j.id === selectedJobId) || milestones[milestones.length - 1] || null,
|
||||
[milestones, selectedJobId]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedJobId && milestones.length > 0) {
|
||||
setSelectedJobId(milestones[milestones.length - 1].id);
|
||||
}
|
||||
}, [milestones, selectedJobId]);
|
||||
|
||||
const dmsTasks = Object.keys(pending?.projects?.dms?.task_defs || {});
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="grid-2">
|
||||
<div className="panel">
|
||||
<div className="panel-header"><h2>DMS(YOLO)一键训练/评估/可视化</h2></div>
|
||||
<div className="panel-body">
|
||||
<div className="crud-form">
|
||||
<label className="field">
|
||||
<span>任务</span>
|
||||
<select value={dmsTask} onChange={(e) => setDmsTask(e.target.value)}>
|
||||
{(dmsTasks.length > 0 ? dmsTasks : [dmsTask]).map((t) => <option key={t} value={t}>{t}</option>)}
|
||||
</select>
|
||||
</label>
|
||||
<label className="field">
|
||||
<span>执行轨</span>
|
||||
<select value={dmsTrack} onChange={(e) => setDmsTrack(e.target.value as "platform" | "local")}>
|
||||
<option value="platform">platform(推荐)</option>
|
||||
<option value="local">local</option>
|
||||
</select>
|
||||
</label>
|
||||
<label className="field" style={{ gridColumn: "1 / -1" }}>
|
||||
<span>评估/可视化权重(可选)</span>
|
||||
<input value={dmsWeights} onChange={(e) => setDmsWeights(e.target.value)} placeholder="/path/to/best.pt" />
|
||||
</label>
|
||||
</div>
|
||||
<div className="audit-quick">
|
||||
<button type="button" className="btn btn-sm btn-primary" onClick={() => submit("train_dms", { task: dmsTask, mode: "full", track: dmsTrack })}>一键训练</button>
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => submit("eval_dms", { task: dmsTask, ...(dmsWeights ? { weights: dmsWeights } : {}) })}>一键评估</button>
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => submit("visualize_dms", { task: dmsTask, ...(dmsWeights ? { weights: dmsWeights } : {}) })}>检测可视化</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="panel">
|
||||
<div className="panel-header"><h2>Lane(UFLD)一键训练/评估{LANE_DATA_VIZ_ENABLED ? "/可视化" : ""}</h2></div>
|
||||
<div className="panel-body">
|
||||
<div className="crud-form">
|
||||
<label className="field">
|
||||
<span>执行轨</span>
|
||||
<select value={laneTrack} onChange={(e) => setLaneTrack(e.target.value as "platform" | "local")}>
|
||||
<option value="platform">platform(推荐)</option>
|
||||
<option value="local">local</option>
|
||||
</select>
|
||||
</label>
|
||||
<label className="field">
|
||||
<span>test_list</span>
|
||||
<input value={laneTestList} onChange={(e) => setLaneTestList(e.target.value)} placeholder="list/test_gt.txt" />
|
||||
</label>
|
||||
<label className="field" style={{ gridColumn: "1 / -1" }}>
|
||||
<span>模型路径(必填用于评估/可视化)</span>
|
||||
<input value={laneModelPath} onChange={(e) => setLaneModelPath(e.target.value)} placeholder="/path/to/best.pth" />
|
||||
</label>
|
||||
<label className="field" style={{ gridColumn: "1 / -1" }}>
|
||||
<span>data_root(可选)</span>
|
||||
<input value={laneDataRoot} onChange={(e) => setLaneDataRoot(e.target.value)} placeholder="/path/to/lane/dataset/root" />
|
||||
</label>
|
||||
</div>
|
||||
<div className="audit-quick">
|
||||
<button type="button" className="btn btn-sm btn-primary" onClick={() => submit("train_lane", { track: laneTrack })}>一键训练</button>
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => submit("eval_lane", { model_path: laneModelPath, ...(laneDataRoot ? { data_root: laneDataRoot } : {}), test_list: laneTestList })}>一键评估</button>
|
||||
{LANE_DATA_VIZ_ENABLED && (
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => submit("visualize_lane", { model_path: laneModelPath, ...(laneDataRoot ? { data_root: laneDataRoot } : {}), test_list: laneTestList })}>检测可视化</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="panel">
|
||||
<div className="panel-header"><h2>训练数据包状态</h2></div>
|
||||
<div className="panel-body pack-list">
|
||||
{packRows.map((p) => (
|
||||
<div key={`${p.project}/${p.name}`} className={`pack-row ${p.enabled ? "active-pack" : ""}`}>
|
||||
<span className="pack-name">{p.project}/{p.name}</span>
|
||||
<span className={`badge ${p.enabled ? "badge-staged" : "badge-idle"}`}>{p.enabled ? "已启用" : "未启用"}</span>
|
||||
<code className="text-sm">python as.py enable {p.project} {p.name}</code>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="panel">
|
||||
<div className="panel-header"><h2>训练里程碑节点(可追溯)</h2></div>
|
||||
<div className="panel-body">
|
||||
{milestones.length === 0 ? (
|
||||
<div className="empty-state">暂无训练/评估节点</div>
|
||||
) : (
|
||||
<div className="milestone-layout">
|
||||
<div className="milestone-track">
|
||||
{milestones.map((j) => {
|
||||
const action = j.action as IterAction;
|
||||
const meta = ACTION_META[action];
|
||||
const weight = extractWeight(j);
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
key={j.id}
|
||||
className={`milestone-node ${selectedJob?.id === j.id ? "active" : ""}`}
|
||||
onClick={() => setSelectedJobId(j.id)}
|
||||
>
|
||||
<div className="milestone-dot" />
|
||||
<div className="milestone-body">
|
||||
<div className="milestone-title">{meta?.label || j.action}</div>
|
||||
<div className="milestone-sub">{fmtTime(j.started_at || j.created_at)}</div>
|
||||
<div className="milestone-sub">
|
||||
<span className={`badge ${statusBadgeClass(j.status)}`}>{j.status}</span>
|
||||
<span>{meta?.project || "未知"}</span>
|
||||
<span>{meta?.kind || "动作"}</span>
|
||||
</div>
|
||||
<div className="milestone-sub mono">{weight ? `权重: ${weight}` : "权重: —"}</div>
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
{selectedJob && (
|
||||
<div className="milestone-detail">
|
||||
<div className="milestone-detail-head">
|
||||
<h3>{ACTION_META[selectedJob.action as IterAction]?.label || selectedJob.action}</h3>
|
||||
<span className={`badge ${statusBadgeClass(selectedJob.status)}`}>{selectedJob.status}</span>
|
||||
</div>
|
||||
<dl className="detail-dl">
|
||||
<dt>Job ID</dt>
|
||||
<dd className="mono">{selectedJob.id}</dd>
|
||||
<dt>创建时间</dt>
|
||||
<dd>{fmtTime(selectedJob.created_at)}</dd>
|
||||
<dt>开始时间</dt>
|
||||
<dd>{fmtTime(selectedJob.started_at)}</dd>
|
||||
<dt>结束时间</dt>
|
||||
<dd>{fmtTime(selectedJob.finished_at)}</dd>
|
||||
<dt>关键权重</dt>
|
||||
<dd className="path-dd">{extractWeight(selectedJob) || "—"}</dd>
|
||||
<dt>执行参数</dt>
|
||||
<dd><pre className="params-pre">{JSON.stringify(selectedJob.params || {}, null, 2)}</pre></dd>
|
||||
<dt>执行结果</dt>
|
||||
<dd><pre className="params-pre milestone-result">{JSON.stringify(selectedJob.result || {}, null, 2)}</pre></dd>
|
||||
</dl>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
33
platform/web/src/pages/Jobs.tsx
Normal file
33
platform/web/src/pages/Jobs.tsx
Normal file
@@ -0,0 +1,33 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { api, type JobRecord } from "../api/client";
|
||||
|
||||
export function JobsPage() {
|
||||
const [items, setItems] = useState<JobRecord[]>([]);
|
||||
useEffect(() => { api.listJobs().then((d) => setItems(d.items || [])); }, []);
|
||||
|
||||
const badge = (s: string) =>
|
||||
({ queued: "badge-pending", running: "badge-training", succeeded: "badge-evaluated", failed: "badge-pending" }[s] || "badge-idle");
|
||||
|
||||
return (
|
||||
<div className="panel">
|
||||
<div className="panel-header"><h2>Job 队列</h2><span className="text-dim">{items.length} 项</span></div>
|
||||
<div className="panel-body table-wrap">
|
||||
<table className="data-table">
|
||||
<thead><tr><th>Job ID</th><th>动作</th><th>状态</th><th>审核单</th><th>开始</th><th>结果</th></tr></thead>
|
||||
<tbody>
|
||||
{items.map((j) => (
|
||||
<tr key={j.id}>
|
||||
<td className="mono text-sm">{j.id}</td>
|
||||
<td>{j.action}</td>
|
||||
<td><span className={`badge ${badge(j.status)}`}>{j.status}</span></td>
|
||||
<td>{j.approval_id || "—"}</td>
|
||||
<td className="text-sm">{j.started_at?.slice(0, 19)}</td>
|
||||
<td className="text-sm">{j.result?.error || (j.result?.ok ? "ok" : "")}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
161
platform/web/src/pages/Labeling.tsx
Normal file
161
platform/web/src/pages/Labeling.tsx
Normal file
@@ -0,0 +1,161 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { useOutletContext } from "react-router-dom";
|
||||
import { api, type BatchRecord, type PendingReport } from "../api/client";
|
||||
import { forStage } from "../lib/labeling";
|
||||
import { useToast } from "../components/Toast";
|
||||
import { UploadsPage } from "./Uploads";
|
||||
|
||||
type Ctx = { refreshMeta: () => void };
|
||||
|
||||
export function LabelingPage() {
|
||||
const { refreshMeta } = useOutletContext<Ctx>();
|
||||
const toast = useToast();
|
||||
const [pending, setPending] = useState<PendingReport | null>(null);
|
||||
const [project, setProject] = useState<"dms" | "lane">("dms");
|
||||
const [selectedTask, setSelectedTask] = useState<string | null>(null);
|
||||
const [selectedPack, setSelectedPack] = useState<string | null>(null);
|
||||
const [selectedBatchId, setSelectedBatchId] = useState<string | null>(null);
|
||||
|
||||
const load = async () => {
|
||||
setPending(await api.pending());
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
}, []);
|
||||
|
||||
const batches = useMemo(() => {
|
||||
const all = pending?.batches || [];
|
||||
if (project === "dms" && selectedTask) return all.filter((b) => b.project === "dms" && b.task === selectedTask);
|
||||
if (project === "lane" && selectedPack) {
|
||||
return all.filter((b) => b.project === "lane" && (b.pack === selectedPack || b.batch === selectedPack));
|
||||
}
|
||||
return all;
|
||||
}, [pending, project, selectedTask, selectedPack]);
|
||||
|
||||
const selected = batches.find((b) => `${b.location}:${b.batch}` === selectedBatchId) || batches[0];
|
||||
|
||||
if (!pending) return <p className="empty-state">加载中…</p>;
|
||||
|
||||
const bs = pending?.batches || [];
|
||||
const dmsTasks = Object.keys(pending.projects?.dms?.task_defs || {});
|
||||
const lanePacks = Object.keys(pending.projects?.lane?.packs || {});
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="kpi-strip">
|
||||
<div className="kpi"><span className="kpi-val">{bs.filter((b) => b.stage === "returned").length}</span><span className="kpi-lbl">待审核入库</span></div>
|
||||
<div className="kpi"><span className="kpi-val">{bs.filter((b) => b.stage === "raw_pool").length}</span><span className="kpi-lbl">待标注原图</span></div>
|
||||
<div className="kpi"><span className="kpi-val">{bs.filter((b) => b.stage === "out_for_labeling").length}</span><span className="kpi-lbl">送标中</span></div>
|
||||
<div className="kpi"><span className="kpi-val">{(pending?.projects?.dms?.active_packs || []).length}</span><span className="kpi-lbl">启用训练包</span></div>
|
||||
</div>
|
||||
|
||||
<div className="panel">
|
||||
<div className="panel-header"><h2>数据上传与自动分析</h2></div>
|
||||
<div className="panel-body">
|
||||
<UploadsPage embedded />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="panel">
|
||||
<div className="panel-header">
|
||||
<h2>待处理批次</h2>
|
||||
<div className="audit-filters">
|
||||
<button type="button" className={`btn btn-sm ${project === "dms" ? "btn-primary" : "btn-ghost"}`} onClick={() => { setProject("dms"); setSelectedPack(null); }}>DMS</button>
|
||||
<button type="button" className={`btn btn-sm ${project === "lane" ? "btn-primary" : "btn-ghost"}`} onClick={() => { setProject("lane"); setSelectedTask(null); }}>Lane</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="panel-body">
|
||||
<div className="catalog-toolbar">
|
||||
<label className="field">
|
||||
<span>{project === "dms" ? "任务" : "数据包"}</span>
|
||||
<select
|
||||
value={project === "dms" ? (selectedTask || "") : (selectedPack || "")}
|
||||
onChange={(e) => {
|
||||
const value = e.target.value;
|
||||
if (project === "dms") setSelectedTask(value || null);
|
||||
else setSelectedPack(value || null);
|
||||
setSelectedBatchId(null);
|
||||
}}
|
||||
>
|
||||
<option value="">全部</option>
|
||||
{(project === "dms" ? dmsTasks : lanePacks).map((name) => (
|
||||
<option key={name} value={name}>{name}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
<div className="grid-2-equal">
|
||||
<div className="panel panel-compact">
|
||||
<div className="panel-header"><h2>批次列表</h2><span className="text-dim">{batches.length} 项</span></div>
|
||||
<div className="panel-body table-wrap">
|
||||
<table className="data-table">
|
||||
<thead><tr><th>批次</th><th>阶段</th><th>位置</th><th>图</th><th>标注</th></tr></thead>
|
||||
<tbody>
|
||||
{batches.map((b) => {
|
||||
const st = forStage(b.stage);
|
||||
const id = `${b.location}:${b.batch}`;
|
||||
return (
|
||||
<tr key={id} className={selected === b ? "row-active" : ""} onClick={() => setSelectedBatchId(id)} style={{ cursor: "pointer" }}>
|
||||
<td><strong>{b.batch}</strong></td>
|
||||
<td><span className={`badge ${st.badge}`}>{st.label}</span></td>
|
||||
<td>{b.location}</td>
|
||||
<td>{b.counts?.images ?? "—"}</td>
|
||||
<td>{b.counts?.labels ?? "—"}</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<div className="panel panel-compact">
|
||||
<div className="panel-header"><h2>详情与动作</h2></div>
|
||||
<div className="panel-body">
|
||||
{selected
|
||||
? <BatchDetail b={selected} onDone={() => { load(); refreshMeta(); toast("已提交"); }} />
|
||||
: <p className="empty-state">请选择批次</p>}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function BatchDetail({ b, onDone }: { b: BatchRecord; onDone: () => void }) {
|
||||
const st = forStage(b.stage);
|
||||
|
||||
return (
|
||||
<div className="batch-detail">
|
||||
<div className="batch-detail-header">
|
||||
<h3>{b.batch}</h3>
|
||||
<span className={`badge ${st.badge}`}>{st.label}</span>
|
||||
</div>
|
||||
<dl className="detail-dl">
|
||||
<dt>数据路径</dt><dd className="mono path-dd">{b.path || "由上传解析后生成"}</dd>
|
||||
<dt>位置</dt><dd>{b.location}{b.pack ? ` · ${b.pack}` : ""}</dd>
|
||||
<dt>图像 / 标注</dt><dd>{b.counts?.images ?? "—"} / {b.counts?.labels ?? "—"}</dd>
|
||||
</dl>
|
||||
{b.next_cli && (
|
||||
<div className="cli-box">
|
||||
<code>{b.next_cli}</code>
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => navigator.clipboard.writeText(b.next_cli!)}>复制命令</button>
|
||||
</div>
|
||||
)}
|
||||
<div className="batch-actions">
|
||||
{b.project === "dms" && b.stage === "returned" && (
|
||||
<button type="button" className="btn btn-sm btn-primary" onClick={async () => {
|
||||
await api.submitBuildBatch({ task: b.task, batch: b.batch, pack: b.pack || "dms_v2", location: b.location });
|
||||
onDone();
|
||||
}}>提交入库审核</button>
|
||||
)}
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={async () => {
|
||||
await api.registerBatch({ project: b.project, task: b.task, batch: b.batch, pack: b.pack, location: b.location });
|
||||
onDone();
|
||||
}}>登记 meta(审核)</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
52
platform/web/src/pages/Login.tsx
Normal file
52
platform/web/src/pages/Login.tsx
Normal file
@@ -0,0 +1,52 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { useNavigate } from "react-router-dom";
|
||||
import { useAuth } from "../auth/AuthContext";
|
||||
|
||||
export function LoginPage() {
|
||||
const { user, loginFeishu, loginDev, authConfig } = useAuth();
|
||||
const navigate = useNavigate();
|
||||
const [err, setErr] = useState("");
|
||||
const [name, setName] = useState("开发用户");
|
||||
|
||||
useEffect(() => {
|
||||
if (user) navigate("/labeling", { replace: true });
|
||||
}, [user, navigate]);
|
||||
|
||||
const onDevLogin = async () => {
|
||||
setErr("");
|
||||
try {
|
||||
await loginDev(name);
|
||||
navigate("/labeling");
|
||||
} catch (e) {
|
||||
setErr(String(e));
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="login-page">
|
||||
<div className="login-shell simple">
|
||||
<section className="login-card panel" id="access">
|
||||
<h2>欢迎登录</h2>
|
||||
<p className="text-dim">广东华胥智能技术 · 卡车主动安全 AEB 平台</p>
|
||||
|
||||
{authConfig?.feishu_enabled && (
|
||||
<button type="button" className="btn btn-primary btn-feishu" onClick={loginFeishu}>
|
||||
立即使用飞书登录
|
||||
</button>
|
||||
)}
|
||||
{authConfig?.dev_auth_enabled && (
|
||||
<div className="dev-login">
|
||||
<p className="text-sm text-dim">开发模式(未配置飞书 App)</p>
|
||||
<input type="text" value={name} onChange={(e) => setName(e.target.value)} placeholder="显示名称" />
|
||||
<button type="button" className="btn btn-ghost" onClick={onDevLogin}>开发登录</button>
|
||||
</div>
|
||||
)}
|
||||
{!authConfig?.feishu_enabled && !authConfig?.dev_auth_enabled && (
|
||||
<p className="empty-state">当前未启用可用登录方式,请联系管理员。</p>
|
||||
)}
|
||||
{err && <p className="login-err">{err}</p>}
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
30
platform/web/src/pages/Logs.tsx
Normal file
30
platform/web/src/pages/Logs.tsx
Normal file
@@ -0,0 +1,30 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { api, type PendingReport } from "../api/client";
|
||||
|
||||
export function LogsPage() {
|
||||
const [pending, setPending] = useState<PendingReport | null>(null);
|
||||
useEffect(() => { api.pending().then(setPending); }, []);
|
||||
|
||||
const recent = pending?.projects?.dms?.recent_ingest || [];
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="panel">
|
||||
<div className="panel-header"><h2>ingest_log.jsonl(最近)</h2></div>
|
||||
<div className="panel-body log-list">
|
||||
{recent.length ? recent.map((l, i) => (
|
||||
<div key={i} className="log-entry">
|
||||
<span className="log-time">{l.ts}</span>
|
||||
<span className="log-type ingest">ingest</span>
|
||||
<span>task={l.task} pack={l.pack} added={l.added ?? "—"}</span>
|
||||
</div>
|
||||
)) : <p className="empty-state">暂无记录</p>}
|
||||
</div>
|
||||
</div>
|
||||
<div className="panel">
|
||||
<div className="panel-header"><h2>approval_queue.jsonl</h2></div>
|
||||
<div className="panel-body"><p className="mono text-sm">HSAP/manifests/approval_queue.jsonl</p></div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
588
platform/web/src/pages/Training.tsx
Normal file
588
platform/web/src/pages/Training.tsx
Normal file
@@ -0,0 +1,588 @@
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { Link, useOutletContext } from "react-router-dom";
|
||||
import {
|
||||
api,
|
||||
type ModelRegistry,
|
||||
type PendingReport,
|
||||
type TrainingRecord,
|
||||
} from "../api/client";
|
||||
import { useToast } from "../components/Toast";
|
||||
import { LANE_DATA_VIZ_ENABLED } from "../config/featureFlags";
|
||||
|
||||
type Ctx = { refreshMeta: () => void };
|
||||
|
||||
type TrainAction =
|
||||
| "train_dms"
|
||||
| "train_lane"
|
||||
| "eval_dms"
|
||||
| "eval_lane"
|
||||
| "visualize_dms"
|
||||
| "visualize_lane"
|
||||
| "promote_dms";
|
||||
|
||||
const CREATE_ACTIONS: { id: TrainAction; label: string; project: "dms" | "lane"; kind: string }[] = [
|
||||
{ id: "train_dms", label: "DMS 训练", project: "dms", kind: "train" },
|
||||
{ id: "eval_dms", label: "DMS 评估", project: "dms", kind: "eval" },
|
||||
{ id: "visualize_dms", label: "DMS 可视化", project: "dms", kind: "visualize" },
|
||||
{ id: "promote_dms", label: "DMS 晋级", project: "dms", kind: "promote" },
|
||||
{ id: "train_lane", label: "Lane 训练", project: "lane", kind: "train" },
|
||||
{ id: "eval_lane", label: "Lane 评估", project: "lane", kind: "eval" },
|
||||
...(LANE_DATA_VIZ_ENABLED
|
||||
? [{ id: "visualize_lane" as const, label: "Lane 可视化", project: "lane" as const, kind: "visualize" }]
|
||||
: []),
|
||||
];
|
||||
|
||||
function statusBadge(status: string): string {
|
||||
if (status === "succeeded") return "badge-promoted";
|
||||
if (status === "running") return "badge-training";
|
||||
if (status === "failed") return "badge-pending";
|
||||
if (status === "queued") return "badge-pending";
|
||||
return "badge-idle";
|
||||
}
|
||||
|
||||
function kindLabel(kind: string): string {
|
||||
return (
|
||||
{
|
||||
train: "训练",
|
||||
eval: "评估",
|
||||
visualize: "可视化",
|
||||
promote: "晋级",
|
||||
pipeline: "流水线",
|
||||
}[kind] || kind
|
||||
);
|
||||
}
|
||||
|
||||
function fmtTime(ts?: string | null): string {
|
||||
if (!ts) return "—";
|
||||
const d = new Date(ts);
|
||||
if (Number.isNaN(d.getTime())) return ts;
|
||||
return d.toLocaleString("zh-CN", { hour12: false });
|
||||
}
|
||||
|
||||
function fmtDuration(sec?: number | null): string {
|
||||
if (sec == null) return "—";
|
||||
if (sec < 60) return `${sec.toFixed(1)}s`;
|
||||
const m = Math.floor(sec / 60);
|
||||
const s = Math.round(sec % 60);
|
||||
return `${m}m ${s}s`;
|
||||
}
|
||||
|
||||
function fmtMetric(metrics: Record<string, unknown>): string {
|
||||
const parts: string[] = [];
|
||||
if (metrics.map50 != null) parts.push(`mAP50=${metrics.map50}`);
|
||||
if (metrics.delta_map50 != null) parts.push(`Δ=${metrics.delta_map50}`);
|
||||
return parts.length > 0 ? parts.join(" · ") : "—";
|
||||
}
|
||||
|
||||
export function TrainingPage() {
|
||||
const { refreshMeta } = useOutletContext<Ctx>();
|
||||
const toast = useToast();
|
||||
const [pending, setPending] = useState<PendingReport | null>(null);
|
||||
const [records, setRecords] = useState<TrainingRecord[]>([]);
|
||||
const [summary, setSummary] = useState({ total: 0, running: 0, queued: 0, succeeded: 0, failed: 0 });
|
||||
const [models, setModels] = useState<ModelRegistry | null>(null);
|
||||
const [selectedId, setSelectedId] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
const [filterProject, setFilterProject] = useState<"" | "dms" | "lane">("");
|
||||
const [filterKind, setFilterKind] = useState("");
|
||||
const [filterStatus, setFilterStatus] = useState("");
|
||||
const [filterTask, setFilterTask] = useState("");
|
||||
|
||||
const [createAction, setCreateAction] = useState<TrainAction>("train_dms");
|
||||
const [createNote, setCreateNote] = useState("");
|
||||
const [dmsTask, setDmsTask] = useState("dam");
|
||||
const [dmsTrack, setDmsTrack] = useState<"platform" | "local">("platform");
|
||||
const [dmsWeights, setDmsWeights] = useState("");
|
||||
const [laneTrack, setLaneTrack] = useState<"platform" | "local">("platform");
|
||||
const [laneModelPath, setLaneModelPath] = useState("");
|
||||
const [laneDataRoot, setLaneDataRoot] = useState("");
|
||||
const [laneTestList, setLaneTestList] = useState("list/test_gt.txt");
|
||||
|
||||
const loadAll = useCallback(async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const [p, list, reg] = await Promise.all([
|
||||
api.pending(),
|
||||
api.listTrainingRecords({
|
||||
project: filterProject || undefined,
|
||||
kind: filterKind || undefined,
|
||||
status: filterStatus || undefined,
|
||||
task: filterTask || undefined,
|
||||
limit: 200,
|
||||
}),
|
||||
api.getModelRegistry("dms", filterTask || undefined),
|
||||
]);
|
||||
setPending(p);
|
||||
setRecords(list.items || []);
|
||||
setSummary(list.summary || { total: 0, running: 0, queued: 0, succeeded: 0, failed: 0 });
|
||||
setModels(reg);
|
||||
} catch (e) {
|
||||
toast(String(e), true);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [filterProject, filterKind, filterStatus, filterTask, toast]);
|
||||
|
||||
useEffect(() => {
|
||||
loadAll();
|
||||
}, [loadAll]);
|
||||
|
||||
useEffect(() => {
|
||||
const tasks = Object.keys(pending?.projects?.dms?.task_defs || {});
|
||||
if (tasks.length > 0 && !tasks.includes(dmsTask)) {
|
||||
setDmsTask(tasks[0]);
|
||||
}
|
||||
}, [pending, dmsTask]);
|
||||
|
||||
useEffect(() => {
|
||||
const hasRunning = records.some((r) => r.status === "running" || r.status === "queued");
|
||||
if (!hasRunning) return;
|
||||
const t = setInterval(loadAll, 5000);
|
||||
return () => clearInterval(t);
|
||||
}, [records, loadAll]);
|
||||
|
||||
const selected = useMemo(
|
||||
() => records.find((r) => r.id === selectedId) || records[0] || null,
|
||||
[records, selectedId]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedId && records.length > 0) {
|
||||
setSelectedId(records[0].id);
|
||||
}
|
||||
}, [records, selectedId]);
|
||||
|
||||
const dmsTasks = Object.keys(pending?.projects?.dms?.task_defs || {});
|
||||
const createMeta = CREATE_ACTIONS.find((a) => a.id === createAction);
|
||||
|
||||
const buildCreateParams = (): Record<string, unknown> => {
|
||||
switch (createAction) {
|
||||
case "train_dms":
|
||||
return { task: dmsTask, mode: "full", track: dmsTrack };
|
||||
case "eval_dms":
|
||||
return { task: dmsTask, ...(dmsWeights ? { weights: dmsWeights } : {}) };
|
||||
case "visualize_dms":
|
||||
return { task: dmsTask, ...(dmsWeights ? { weights: dmsWeights } : {}) };
|
||||
case "promote_dms":
|
||||
return { task: dmsTask };
|
||||
case "train_lane":
|
||||
return { track: laneTrack };
|
||||
case "eval_lane":
|
||||
return {
|
||||
model_path: laneModelPath,
|
||||
test_list: laneTestList,
|
||||
...(laneDataRoot ? { data_root: laneDataRoot } : {}),
|
||||
};
|
||||
case "visualize_lane":
|
||||
return {
|
||||
model_path: laneModelPath,
|
||||
test_list: laneTestList,
|
||||
...(laneDataRoot ? { data_root: laneDataRoot } : {}),
|
||||
};
|
||||
default:
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
const handleCreate = async () => {
|
||||
if (createAction === "eval_lane" && !laneModelPath.trim()) {
|
||||
toast("Lane 评估需填写模型路径", true);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const approval = await api.createTrainingRecord(createAction, buildCreateParams(), createNote || undefined);
|
||||
toast(`已提交审核单 ${approval.id}`);
|
||||
refreshMeta();
|
||||
await loadAll();
|
||||
setCreateNote("");
|
||||
} catch (e) {
|
||||
toast(String(e), true);
|
||||
}
|
||||
};
|
||||
|
||||
const currentTaskVersion =
|
||||
filterTask && models?.tasks?.[filterTask]
|
||||
? models.tasks[filterTask]
|
||||
: dmsTask && models?.tasks?.[dmsTask]
|
||||
? models.tasks[dmsTask]
|
||||
: null;
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="kpi-strip">
|
||||
<div className="kpi">
|
||||
<span className="kpi-val">{summary.total}</span>
|
||||
<span className="kpi-lbl">训练记录</span>
|
||||
</div>
|
||||
<div className="kpi">
|
||||
<span className="kpi-val">{summary.running + summary.queued}</span>
|
||||
<span className="kpi-lbl">进行中</span>
|
||||
</div>
|
||||
<div className="kpi">
|
||||
<span className="kpi-val">{summary.succeeded}</span>
|
||||
<span className="kpi-lbl">成功</span>
|
||||
</div>
|
||||
<div className="kpi">
|
||||
<span className="kpi-val">{summary.failed}</span>
|
||||
<span className="kpi-lbl">失败</span>
|
||||
</div>
|
||||
<div className="kpi">
|
||||
<span className="kpi-val text-sm mono">{currentTaskVersion?.current ? "有线上" : "—"}</span>
|
||||
<span className="kpi-lbl">DMS 当前模型</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="panel">
|
||||
<div className="panel-header">
|
||||
<h2>新建训练 / 评估</h2>
|
||||
</div>
|
||||
<div className="panel-body">
|
||||
<div className="crud-form">
|
||||
<label className="field">
|
||||
<span>动作</span>
|
||||
<select value={createAction} onChange={(e) => setCreateAction(e.target.value as TrainAction)}>
|
||||
{CREATE_ACTIONS.map((a) => (
|
||||
<option key={a.id} value={a.id}>
|
||||
{a.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<label className="field">
|
||||
<span>备注</span>
|
||||
<input
|
||||
value={createNote}
|
||||
onChange={(e) => setCreateNote(e.target.value)}
|
||||
placeholder="可选:版本说明、实验目的"
|
||||
/>
|
||||
</label>
|
||||
|
||||
{createMeta?.project === "dms" && (
|
||||
<>
|
||||
<label className="field">
|
||||
<span>任务</span>
|
||||
<select value={dmsTask} onChange={(e) => setDmsTask(e.target.value)}>
|
||||
{(dmsTasks.length > 0 ? dmsTasks : [dmsTask]).map((t) => (
|
||||
<option key={t} value={t}>
|
||||
{t}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
{createAction === "train_dms" && (
|
||||
<label className="field">
|
||||
<span>执行轨</span>
|
||||
<select value={dmsTrack} onChange={(e) => setDmsTrack(e.target.value as "platform" | "local")}>
|
||||
<option value="platform">platform</option>
|
||||
<option value="local">local</option>
|
||||
</select>
|
||||
</label>
|
||||
)}
|
||||
{["eval_dms", "visualize_dms"].includes(createAction) && (
|
||||
<label className="field" style={{ gridColumn: "1 / -1" }}>
|
||||
<span>权重路径(可选)</span>
|
||||
<input
|
||||
value={dmsWeights}
|
||||
onChange={(e) => setDmsWeights(e.target.value)}
|
||||
placeholder="/path/to/best.pt"
|
||||
/>
|
||||
</label>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{createMeta?.project === "lane" && (
|
||||
<>
|
||||
{createAction === "train_lane" && (
|
||||
<label className="field">
|
||||
<span>执行轨</span>
|
||||
<select value={laneTrack} onChange={(e) => setLaneTrack(e.target.value as "platform" | "local")}>
|
||||
<option value="platform">platform</option>
|
||||
<option value="local">local</option>
|
||||
</select>
|
||||
</label>
|
||||
)}
|
||||
{createAction !== "train_lane" && (
|
||||
<>
|
||||
<label className="field" style={{ gridColumn: "1 / -1" }}>
|
||||
<span>模型路径</span>
|
||||
<input
|
||||
value={laneModelPath}
|
||||
onChange={(e) => setLaneModelPath(e.target.value)}
|
||||
placeholder="/path/to/best.pth"
|
||||
/>
|
||||
</label>
|
||||
<label className="field">
|
||||
<span>test_list</span>
|
||||
<input value={laneTestList} onChange={(e) => setLaneTestList(e.target.value)} />
|
||||
</label>
|
||||
<label className="field">
|
||||
<span>data_root(可选)</span>
|
||||
<input value={laneDataRoot} onChange={(e) => setLaneDataRoot(e.target.value)} />
|
||||
</label>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
<div className="crud-actions">
|
||||
<button type="button" className="btn btn-primary" onClick={handleCreate}>
|
||||
提交训练任务
|
||||
</button>
|
||||
<span className="text-dim">提交后进入审核队列,批准后开始执行</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="panel">
|
||||
<div className="panel-header">
|
||||
<h2>训练记录</h2>
|
||||
<button type="button" className="btn btn-sm btn-ghost" onClick={() => loadAll()}>
|
||||
刷新
|
||||
</button>
|
||||
</div>
|
||||
<div className="panel-body">
|
||||
<div className="catalog-toolbar">
|
||||
<label className="field">
|
||||
<span>项目</span>
|
||||
<select value={filterProject} onChange={(e) => setFilterProject(e.target.value as "" | "dms" | "lane")}>
|
||||
<option value="">全部</option>
|
||||
<option value="dms">DMS</option>
|
||||
<option value="lane">Lane</option>
|
||||
</select>
|
||||
</label>
|
||||
<label className="field">
|
||||
<span>类型</span>
|
||||
<select value={filterKind} onChange={(e) => setFilterKind(e.target.value)}>
|
||||
<option value="">全部</option>
|
||||
<option value="train">训练</option>
|
||||
<option value="eval">评估</option>
|
||||
<option value="visualize">可视化</option>
|
||||
<option value="promote">晋级</option>
|
||||
</select>
|
||||
</label>
|
||||
<label className="field">
|
||||
<span>状态</span>
|
||||
<select value={filterStatus} onChange={(e) => setFilterStatus(e.target.value)}>
|
||||
<option value="">全部</option>
|
||||
<option value="queued">queued</option>
|
||||
<option value="running">running</option>
|
||||
<option value="succeeded">succeeded</option>
|
||||
<option value="failed">failed</option>
|
||||
</select>
|
||||
</label>
|
||||
<label className="field">
|
||||
<span>DMS 任务</span>
|
||||
<select value={filterTask} onChange={(e) => setFilterTask(e.target.value)}>
|
||||
<option value="">全部</option>
|
||||
{dmsTasks.map((t) => (
|
||||
<option key={t} value={t}>
|
||||
{t}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{loading && records.length === 0 ? (
|
||||
<p className="empty-state">加载中…</p>
|
||||
) : records.length === 0 ? (
|
||||
<p className="empty-state">暂无训练记录,可在上方提交新任务</p>
|
||||
) : (
|
||||
<div className="grid-2-equal">
|
||||
<div className="panel panel-compact">
|
||||
<div className="panel-header">
|
||||
<h2>列表</h2>
|
||||
<span className="text-dim">{records.length} 项</span>
|
||||
</div>
|
||||
<div className="panel-body table-wrap">
|
||||
<table className="data-table compact">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>时间</th>
|
||||
<th>动作</th>
|
||||
<th>任务</th>
|
||||
<th>状态</th>
|
||||
<th>指标</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{records.map((r) => (
|
||||
<tr
|
||||
key={r.id}
|
||||
className={selected?.id === r.id ? "row-active" : ""}
|
||||
style={{ cursor: "pointer" }}
|
||||
onClick={() => setSelectedId(r.id)}
|
||||
>
|
||||
<td className="text-sm">{fmtTime(r.started_at || r.created_at).slice(0, 16)}</td>
|
||||
<td>
|
||||
<div>{r.action_label || r.action}</div>
|
||||
<div className="text-dim">
|
||||
{r.project} · {kindLabel(r.kind || "")}
|
||||
</div>
|
||||
</td>
|
||||
<td>{r.task || "—"}</td>
|
||||
<td>
|
||||
<span className={`badge ${statusBadge(r.status)}`}>{r.status}</span>
|
||||
</td>
|
||||
<td className="text-sm">{fmtMetric(r.metrics || {})}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="panel panel-compact">
|
||||
<div className="panel-header">
|
||||
<h2>详情</h2>
|
||||
</div>
|
||||
<div className="panel-body">
|
||||
{selected ? (
|
||||
<TrainingDetail record={selected} />
|
||||
) : (
|
||||
<p className="empty-state">请选择一条记录</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{models && Object.keys(models.tasks || {}).length > 0 && (
|
||||
<div className="panel">
|
||||
<div className="panel-header">
|
||||
<h2>DMS 模型版本</h2>
|
||||
</div>
|
||||
<div className="panel-body table-wrap">
|
||||
<table className="data-table compact">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>任务</th>
|
||||
<th>候选 (candidate)</th>
|
||||
<th>线上 (current)</th>
|
||||
<th>最近评估</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{Object.entries(models.tasks || {}).map(([task, ver]) => {
|
||||
const v = ver as Record<string, unknown>;
|
||||
const lastEval = (v.last_eval || {}) as Record<string, unknown>;
|
||||
return (
|
||||
<tr key={task}>
|
||||
<td>
|
||||
<strong>{task}</strong>
|
||||
</td>
|
||||
<td className="mono text-sm path-dd">{String(v.candidate || "—")}</td>
|
||||
<td className="mono text-sm path-dd">{String(v.current || "—")}</td>
|
||||
<td className="text-sm">
|
||||
{lastEval.map50 != null ? `mAP50=${lastEval.map50}` : "—"}
|
||||
{lastEval.weights ? (
|
||||
<div className="text-dim mono">{String(lastEval.weights)}</div>
|
||||
) : null}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(models?.eval_history?.length || 0) > 0 && (
|
||||
<div className="panel">
|
||||
<div className="panel-header">
|
||||
<h2>评估历史</h2>
|
||||
</div>
|
||||
<div className="panel-body table-wrap">
|
||||
<table className="data-table compact">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>时间</th>
|
||||
<th>任务</th>
|
||||
<th>mAP50</th>
|
||||
<th>Δ</th>
|
||||
<th>权重</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(models?.eval_history || []).map((row, i) => (
|
||||
<tr key={`${row.ts}-${i}`}>
|
||||
<td className="text-sm">{fmtTime(row.ts as string)}</td>
|
||||
<td>{String(row.task || "—")}</td>
|
||||
<td>{row.map50 != null ? String(row.map50) : "—"}</td>
|
||||
<td>{row.delta_map50 != null ? String(row.delta_map50) : "—"}</td>
|
||||
<td className="mono text-sm path-dd">{String(row.weights || "—")}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function TrainingDetail({ record }: { record: TrainingRecord }) {
|
||||
const approval = record.approval;
|
||||
|
||||
return (
|
||||
<div className="batch-detail">
|
||||
<div className="batch-detail-header">
|
||||
<h3>{record.action_label || record.action}</h3>
|
||||
<span className={`badge ${statusBadge(record.status)}`}>{record.status}</span>
|
||||
</div>
|
||||
<dl className="detail-dl">
|
||||
<dt>Job ID</dt>
|
||||
<dd className="mono">{record.id}</dd>
|
||||
<dt>项目 / 类型</dt>
|
||||
<dd>
|
||||
{record.project} · {kindLabel(record.kind || "")}
|
||||
{record.task ? ` · ${record.task}` : ""}
|
||||
{record.track ? ` · ${record.track}` : ""}
|
||||
</dd>
|
||||
<dt>创建 / 开始 / 结束</dt>
|
||||
<dd>
|
||||
{fmtTime(record.created_at)} → {fmtTime(record.started_at)} → {fmtTime(record.finished_at)}
|
||||
</dd>
|
||||
<dt>耗时</dt>
|
||||
<dd>{fmtDuration(record.duration_sec)}</dd>
|
||||
<dt>权重 / 产物</dt>
|
||||
<dd className="path-dd mono">{record.weight_path || "—"}</dd>
|
||||
<dt>指标</dt>
|
||||
<dd>{fmtMetric(record.metrics || {})}</dd>
|
||||
{record.error && (
|
||||
<>
|
||||
<dt>错误</dt>
|
||||
<dd className="text-sm" style={{ color: "var(--danger)" }}>
|
||||
{record.error}
|
||||
</dd>
|
||||
</>
|
||||
)}
|
||||
{approval && (
|
||||
<>
|
||||
<dt>审核单</dt>
|
||||
<dd>
|
||||
<Link to={`/audit/${approval.id}`}>{approval.id}</Link>
|
||||
<span className="text-dim"> · {approval.status}</span>
|
||||
{approval.note ? <div className="text-sm">{approval.note}</div> : null}
|
||||
</dd>
|
||||
</>
|
||||
)}
|
||||
<dt>执行参数</dt>
|
||||
<dd>
|
||||
<pre className="params-pre">{JSON.stringify(record.params || {}, null, 2)}</pre>
|
||||
</dd>
|
||||
<dt>执行结果</dt>
|
||||
<dd>
|
||||
<pre className="params-pre milestone-result">{JSON.stringify(record.result || {}, null, 2)}</pre>
|
||||
</dd>
|
||||
</dl>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
163
platform/web/src/pages/Uploads.tsx
Normal file
163
platform/web/src/pages/Uploads.tsx
Normal file
@@ -0,0 +1,163 @@
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { api, type DataCandidate } from "../api/client";
|
||||
import { useToast } from "../components/Toast";
|
||||
|
||||
export function UploadsPage({ embedded = false }: { embedded?: boolean }) {
|
||||
const toast = useToast();
|
||||
const [project, setProject] = useState<"dms" | "lane">("dms");
|
||||
const [task, setTask] = useState("dam");
|
||||
const [tasks, setTasks] = useState<string[]>([]);
|
||||
const [file, setFile] = useState<File | null>(null);
|
||||
const [progress, setProgress] = useState(0);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
const [candidates, setCandidates] = useState<DataCandidate[]>([]);
|
||||
const [activeId, setActiveId] = useState<string | null>(null);
|
||||
|
||||
const loadCandidates = async () => {
|
||||
const res = await api.listDataCandidates(80);
|
||||
setCandidates(res.items || []);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
loadCandidates();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const loadTasks = async () => {
|
||||
try {
|
||||
const pending = await api.pending();
|
||||
const defs = pending.projects?.dms?.task_defs || {};
|
||||
const names = Object.keys(defs);
|
||||
setTasks(names);
|
||||
if (names.length > 0 && !names.includes(task)) {
|
||||
setTask(names[0]);
|
||||
}
|
||||
} catch {
|
||||
// keep manual fallback value
|
||||
}
|
||||
};
|
||||
loadTasks();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!activeId) return;
|
||||
const t = setInterval(async () => {
|
||||
try {
|
||||
const latest = await api.getDataCandidate(activeId);
|
||||
setCandidates((prev) => [latest, ...prev.filter((x) => x.id !== latest.id)]);
|
||||
if (latest.status === "analyzed" || latest.status === "failed") {
|
||||
setActiveId(null);
|
||||
await loadCandidates();
|
||||
}
|
||||
} catch {
|
||||
// keep polling loop running; backend errors are surfaced in status list
|
||||
}
|
||||
}, 2000);
|
||||
return () => clearInterval(t);
|
||||
}, [activeId]);
|
||||
|
||||
const submit = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
if (!file) {
|
||||
toast("请先选择压缩包文件");
|
||||
return;
|
||||
}
|
||||
setUploading(true);
|
||||
setProgress(0);
|
||||
try {
|
||||
const res = await api.uploadDatasetFile(file, project, project === "dms" ? task : undefined, setProgress);
|
||||
setActiveId(res.candidate.id);
|
||||
setCandidates((prev) => [res.candidate, ...prev.filter((x) => x.id !== res.candidate.id)]);
|
||||
toast(`上传成功,开始分析:${res.candidate.id}`);
|
||||
} catch (err) {
|
||||
toast(err instanceof Error ? err.message : "上传失败");
|
||||
} finally {
|
||||
setUploading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const active = useMemo(() => candidates.find((x) => x.id === activeId) || null, [candidates, activeId]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className={embedded ? "" : "panel"}>
|
||||
<div className="panel-header"><h2>上传候选数据包</h2></div>
|
||||
<div className="panel-body">
|
||||
<form className="crud-form" onSubmit={submit}>
|
||||
<label className="field">
|
||||
<span>Project</span>
|
||||
<select value={project} onChange={(e) => {
|
||||
const next = e.target.value as "dms" | "lane";
|
||||
setProject(next);
|
||||
if (next === "dms" && tasks.length > 0 && !tasks.includes(task)) {
|
||||
setTask(tasks[0]);
|
||||
}
|
||||
}}>
|
||||
<option value="dms">dms</option>
|
||||
<option value="lane">lane</option>
|
||||
</select>
|
||||
</label>
|
||||
{project === "dms" && (
|
||||
<label className="field">
|
||||
<span>Task</span>
|
||||
<select value={task} onChange={(e) => setTask(e.target.value)}>
|
||||
{(tasks.length > 0 ? tasks : [task]).map((t) => (
|
||||
<option key={t} value={t}>{t}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
)}
|
||||
<label className="field" style={{ gridColumn: "1 / -1" }}>
|
||||
<span>压缩包</span>
|
||||
<input type="file" accept=".zip,.tar,.tar.gz,.tgz" onChange={(e) => setFile(e.target.files?.[0] || null)} />
|
||||
</label>
|
||||
<div className="crud-actions">
|
||||
<button type="submit" className="btn btn-primary" disabled={uploading}>
|
||||
{uploading ? "上传中..." : "上传并自动分析"}
|
||||
</button>
|
||||
<button type="button" className="btn btn-ghost" onClick={() => loadCandidates()}>刷新</button>
|
||||
</div>
|
||||
</form>
|
||||
<div style={{ marginTop: 12 }}>
|
||||
<div className="progress-bar"><div className="progress-fill" style={{ width: `${progress}%` }} /></div>
|
||||
<p className="audit-note">上传进度:{progress}% {active ? `| 当前分析状态:${active.status}` : ""}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className={embedded ? "panel panel-compact" : "panel"}>
|
||||
<div className="panel-header"><h2>候选数据与分析状态</h2></div>
|
||||
<div className="panel-body table-wrap">
|
||||
<table className="data-table compact">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>项目/任务</th>
|
||||
<th>状态</th>
|
||||
<th>格式</th>
|
||||
<th>train/val/test</th>
|
||||
<th>错误</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{candidates.map((c) => (
|
||||
<tr key={c.id} className={c.id === activeId ? "row-active" : ""}>
|
||||
<td className="mono">{c.id}</td>
|
||||
<td>{c.project}{c.task ? `/${c.task}` : ""}</td>
|
||||
<td><span className="badge badge-idle">{c.status}</span></td>
|
||||
<td>{c.format_id || "—"}</td>
|
||||
<td>
|
||||
{c.split_counts
|
||||
? `${c.split_counts.train || 0}/${c.split_counts.val || 0}/${c.split_counts.test || 0}`
|
||||
: "—"}
|
||||
</td>
|
||||
<td>{c.error_message || "—"}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
2039
platform/web/src/styles/main.css
Normal file
2039
platform/web/src/styles/main.css
Normal file
File diff suppressed because it is too large
Load Diff
9
platform/web/src/vite-env.d.ts
vendored
Normal file
9
platform/web/src/vite-env.d.ts
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
/// <reference types="vite/client" />
|
||||
|
||||
interface ImportMetaEnv {
|
||||
readonly VITE_API_BASE: string;
|
||||
}
|
||||
|
||||
interface ImportMeta {
|
||||
readonly env: ImportMetaEnv;
|
||||
}
|
||||
21
platform/web/tsconfig.json
Normal file
21
platform/web/tsconfig.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"isolatedModules": true,
|
||||
"moduleDetection": "force",
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedSideEffectImports": true
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
22
platform/web/vite.config.ts
Normal file
22
platform/web/vite.config.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { defineConfig, loadEnv } from "vite";
|
||||
import react from "@vitejs/plugin-react";
|
||||
|
||||
export default defineConfig(({ mode }) => {
|
||||
const env = loadEnv(mode, process.cwd(), "");
|
||||
const proxyTarget = env.VITE_API_PROXY || "http://127.0.0.1:8787";
|
||||
|
||||
return {
|
||||
plugins: [react()],
|
||||
server: {
|
||||
host: true,
|
||||
port: 5173,
|
||||
proxy: {
|
||||
"/api": { target: proxyTarget, changeOrigin: true },
|
||||
},
|
||||
},
|
||||
build: {
|
||||
outDir: "dist",
|
||||
emptyOutDir: true,
|
||||
},
|
||||
};
|
||||
});
|
||||
Reference in New Issue
Block a user