feat: HSAP platform v2 — modular navigation, quality review, audit log, world model simulation
Major changes: - New frontend (platform/web/): Vite + React 18 + TypeScript + Tailwind - 4-module navigation: 数据送标 / 模型管理 / 车队管理 / 系统管理 - Data catalog with charts (DMS/ADAS/Lane 3-tab view) - Quality review workflow (标注质检): Good/Fine/Bad scoring with auto-advance - Audit enhancements: batch operations, rejection categories, Feishu notifications - Operation audit log (操作日志) - World model simulation studio (仿真工坊) - Dataset version management with snapshots and diff - ADAS 7-class dataset integration (138K images organized + compressed) - User management with Feishu integration and pagination - CRUD/search/filter on all pages, card layout redesign - PIL-optimized image overlay rendering - Auto-snapshot on build, in_review workflow stage - Removed embedded algorithm code (now in workspace)
This commit is contained in:
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
from typing import Annotated, Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import RedirectResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -12,8 +12,8 @@ from sqlalchemy.orm import Session
|
||||
from as_platform.auth.deps import get_current_user, require_permission
|
||||
from as_platform.auth.feishu import build_authorize_url, exchange_code, is_feishu_configured, verify_state
|
||||
from as_platform.auth.jwt import create_access_token
|
||||
from as_platform.auth.users import get_or_create_dev_user, list_users, set_user_roles, upsert_feishu_user
|
||||
from as_platform.config import DEV_AUTH_ENABLED, FRONTEND_URL
|
||||
from as_platform.auth.users import get_or_create_dev_user, list_users, list_users_paginated, set_user_roles, upsert_feishu_user
|
||||
from as_platform.config import DEV_AUTH_ENABLED, FORCE_DEV_AUTH, FRONTEND_URL
|
||||
from as_platform.db.engine import get_db
|
||||
from as_platform.db.init_db import user_to_dict
|
||||
from as_platform.db.models import User
|
||||
@@ -33,7 +33,7 @@ class SetRolesBody(BaseModel):
|
||||
def auth_config() -> dict[str, Any]:
|
||||
return {
|
||||
"feishu_enabled": is_feishu_configured(),
|
||||
"dev_auth_enabled": DEV_AUTH_ENABLED and not is_feishu_configured(),
|
||||
"dev_auth_enabled": DEV_AUTH_ENABLED and (FORCE_DEV_AUTH or not is_feishu_configured()),
|
||||
}
|
||||
|
||||
|
||||
@@ -56,6 +56,10 @@ def feishu_callback(code: str, state: str, db: Annotated[Session, Depends(get_db
|
||||
user = upsert_feishu_user(db, info)
|
||||
db.commit()
|
||||
token = create_access_token(user.id)
|
||||
# 审计日志
|
||||
from as_platform.audit.log_utils import log_op
|
||||
log_op(user_id=user.id, user_name=user.name, category="auth", action="login",
|
||||
target_type="user", target_id=str(user.id), summary=f"飞书登录: {user.name}")
|
||||
# 避免 BrowserRouter 直达路径 404,统一回根路径并附带 token
|
||||
qs = urlencode({"token": token})
|
||||
return RedirectResponse(f"{FRONTEND_URL}/?{qs}")
|
||||
@@ -63,11 +67,15 @@ def feishu_callback(code: str, state: str, db: Annotated[Session, Depends(get_db
|
||||
|
||||
@router.post("/dev/login")
|
||||
def dev_login(body: DevLoginBody, db: Annotated[Session, Depends(get_db)]):
|
||||
if not DEV_AUTH_ENABLED or is_feishu_configured():
|
||||
if not DEV_AUTH_ENABLED or (not FORCE_DEV_AUTH and is_feishu_configured()):
|
||||
raise HTTPException(403, "开发登录未启用")
|
||||
user = get_or_create_dev_user(db, body.name)
|
||||
db.commit()
|
||||
token = create_access_token(user.id)
|
||||
# 审计日志
|
||||
from as_platform.audit.log_utils import log_op
|
||||
log_op(user_id=user.id, user_name=user.name, category="auth", action="login",
|
||||
target_type="user", target_id=str(user.id), summary=f"开发登录: {user.name}")
|
||||
return {"access_token": token, "user": user_to_dict(user)}
|
||||
|
||||
|
||||
@@ -80,8 +88,13 @@ def auth_me(user: Annotated[User, Depends(get_current_user)]):
|
||||
def auth_list_users(
|
||||
_user: Annotated[User, Depends(require_permission("admin:users"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
search: str = Query(""),
|
||||
role: str = Query(""),
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
):
|
||||
return {"items": [user_to_dict(u) for u in list_users(db)]}
|
||||
users, total = list_users_paginated(db, search=search, role_code=role, offset=offset, limit=limit)
|
||||
return {"items": [user_to_dict(u) for u in users], "total": total}
|
||||
|
||||
|
||||
@router.put("/users/{user_id}/roles")
|
||||
@@ -95,4 +108,8 @@ def auth_set_roles(
|
||||
if not user:
|
||||
raise HTTPException(404, "用户不存在")
|
||||
db.commit()
|
||||
from as_platform.audit.log_utils import log_op
|
||||
log_op(user_id=_user.id, user_name=_user.name, category="system", action="set_roles",
|
||||
target_type="user", target_id=str(user_id), summary=f"修改角色: {user.name} → {body.role_codes}",
|
||||
detail={"user_id": user_id, "roles": body.role_codes})
|
||||
return user_to_dict(user)
|
||||
|
||||
133
platform/as_platform/api/delivery_routes.py
Normal file
133
platform/as_platform/api/delivery_routes.py
Normal file
@@ -0,0 +1,133 @@
|
||||
"""平台批次送标申请 API。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from as_platform.auth.deps import get_current_user, require_any_permission
|
||||
from as_platform.db.models import User
|
||||
from as_platform.deliveries import service as delivery_svc
|
||||
|
||||
router = APIRouter(prefix="/api/v1/deliveries", tags=["deliveries"])
|
||||
|
||||
|
||||
class DeliveryBody(BaseModel):
|
||||
project: str = "dms"
|
||||
task: str | None = None
|
||||
mode: str | None = None
|
||||
batch_name: str = ""
|
||||
source_type: str | None = None
|
||||
vehicle_scene: str | None = None
|
||||
collection_start: str | None = None
|
||||
collection_end: str | None = None
|
||||
data_path: str = ""
|
||||
estimated_count: int | None = None
|
||||
remark: str | None = None
|
||||
owner_user_id: int | None = None
|
||||
owner_name: str | None = None
|
||||
|
||||
|
||||
class DeliveryPatchBody(BaseModel):
|
||||
project: str | None = None
|
||||
task: str | None = None
|
||||
mode: str | None = None
|
||||
batch_name: str | None = None
|
||||
source_type: str | None = None
|
||||
vehicle_scene: str | None = None
|
||||
collection_start: str | None = None
|
||||
collection_end: str | None = None
|
||||
data_path: str | None = None
|
||||
estimated_count: int | None = None
|
||||
remark: str | None = None
|
||||
owner_user_id: int | None = None
|
||||
owner_name: str | None = None
|
||||
|
||||
|
||||
@router.get("")
|
||||
def api_list_deliveries(
|
||||
_user: Annotated[User, Depends(require_any_permission("read:deliveries", "read:pending", "*"))],
|
||||
status: str | None = Query(None),
|
||||
mine: bool = Query(False),
|
||||
mine_editable: bool = Query(False, description="待我处理:仅草稿/驳回"),
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
mine_id = _user.id if mine or mine_editable else None
|
||||
return delivery_svc.list_deliveries(
|
||||
status=status,
|
||||
mine_user_id=mine_id,
|
||||
mine_editable_only=mine_editable,
|
||||
offset=offset,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{delivery_id}")
|
||||
def api_get_delivery(
|
||||
delivery_id: str,
|
||||
_user: Annotated[User, Depends(require_any_permission("read:deliveries", "read:pending", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
row = delivery_svc.get_delivery(delivery_id)
|
||||
if not row:
|
||||
raise HTTPException(404, "送标申请不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.post("")
|
||||
def api_create_delivery(
|
||||
body: DeliveryBody,
|
||||
user: Annotated[User, Depends(require_any_permission("write:delivery_submit", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return delivery_svc.create_delivery(body.model_dump(), user)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.patch("/{delivery_id}")
|
||||
def api_patch_delivery(
|
||||
delivery_id: str,
|
||||
body: DeliveryPatchBody,
|
||||
user: Annotated[User, Depends(require_any_permission("write:delivery_submit", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
data = body.model_dump(exclude_unset=True)
|
||||
try:
|
||||
return delivery_svc.update_delivery(delivery_id, data, user)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.post("/{delivery_id}/submit")
|
||||
def api_submit_delivery(
|
||||
delivery_id: str,
|
||||
user: Annotated[User, Depends(require_any_permission("write:delivery_submit", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return delivery_svc.submit_delivery_for_review(delivery_id, user)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.post("/{delivery_id}/retry-ingest")
|
||||
def api_retry_delivery_ingest(
|
||||
delivery_id: str,
|
||||
user: Annotated[User, Depends(require_any_permission("write:delivery_submit", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return delivery_svc.retry_delivery_ingest(delivery_id, user)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.delete("/{delivery_id}")
|
||||
def api_delete_delivery(
|
||||
delivery_id: str,
|
||||
user: Annotated[User, Depends(require_any_permission("write:delivery_submit", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
delivery_svc.delete_delivery(delivery_id, user)
|
||||
return {"ok": True}
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
75
platform/as_platform/api/feishu_routes.py
Normal file
75
platform/as_platform/api/feishu_routes.py
Normal file
@@ -0,0 +1,75 @@
|
||||
"""飞书集成 API。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from as_platform.auth.deps import require_any_permission
|
||||
from as_platform.db.models import User
|
||||
from as_platform.config import FEISHU_BITABLE_AUTO_INGEST, FEISHU_BITABLE_WEBHOOK_ENABLED
|
||||
from as_platform.integrations.feishu_bitable import connectivity_check, is_bitable_configured, list_tables
|
||||
from as_platform.integrations.feishu_bitable_ingest import process_pending_ingest
|
||||
from as_platform.integrations.feishu_bitable_sync import backfill_hints
|
||||
from as_platform.integrations.feishu_notify import is_notify_configured
|
||||
from as_platform.jobs.feishu_bitable_sync import run_sync_cycle
|
||||
|
||||
router = APIRouter(prefix="/api/v1/integrations/feishu", tags=["feishu"])
|
||||
|
||||
|
||||
@router.get("/bitable/status")
|
||||
def api_bitable_status(
|
||||
_user: Annotated[User, Depends(require_any_permission("write:labeling_assign", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
return connectivity_check()
|
||||
|
||||
|
||||
@router.get("/bitable/tables")
|
||||
def api_bitable_tables(
|
||||
_user: Annotated[User, Depends(require_any_permission("write:labeling_assign", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
if not is_bitable_configured():
|
||||
return {"items": [], "message": "未配置 BITABLE_APP_TOKEN / TABLE_ID"}
|
||||
return {"items": list_tables()}
|
||||
|
||||
|
||||
@router.post("/bitable/sync")
|
||||
def api_bitable_sync(
|
||||
_user: Annotated[User, Depends(require_any_permission("write:labeling_assign", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
return run_sync_cycle()
|
||||
|
||||
|
||||
@router.post("/bitable/ingest")
|
||||
def api_bitable_ingest(
|
||||
_user: Annotated[User, Depends(require_any_permission("write:labeling_assign", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
"""手动触发「待落盘」入湖(须 FEISHU_BITABLE_AUTO_INGEST=1 或本接口始终可用)。"""
|
||||
return process_pending_ingest()
|
||||
|
||||
|
||||
@router.get("/bitable/config")
|
||||
def api_bitable_config(
|
||||
_user: Annotated[User, Depends(require_any_permission("write:labeling_assign", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"bitable_configured": is_bitable_configured(),
|
||||
"notify_configured": is_notify_configured(),
|
||||
"auto_ingest": FEISHU_BITABLE_AUTO_INGEST,
|
||||
"webhook_enabled": FEISHU_BITABLE_WEBHOOK_ENABLED,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/bitable/webhook")
|
||||
def api_bitable_webhook_disabled() -> dict[str, Any]:
|
||||
"""内网默认关闭;有公网穿透后再实现验签入站。"""
|
||||
if not FEISHU_BITABLE_WEBHOOK_ENABLED:
|
||||
return {"ok": False, "message": "FEISHU_BITABLE_WEBHOOK_ENABLED 未开启"}
|
||||
return {"ok": False, "message": "Webhook 处理器尚未实现"}
|
||||
|
||||
|
||||
@router.get("/bitable/backfill-hints")
|
||||
def api_bitable_backfill_hints(
|
||||
_user: Annotated[User, Depends(require_any_permission("write:labeling_assign", "*"))],
|
||||
) -> dict[str, Any]:
|
||||
return backfill_hints()
|
||||
344
platform/as_platform/api/fleet_routes.py
Normal file
344
platform/as_platform/api/fleet_routes.py
Normal file
@@ -0,0 +1,344 @@
|
||||
"""Fleet map API routes."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Form, Header, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from as_platform.auth.deps import get_current_user, require_permission
|
||||
from as_platform.config import AMAP_KEY, FLEET_MAP_ENABLED, MAP_TILE_PROVIDER, TBOX_INGEST_TOKEN
|
||||
from as_platform.db.engine import get_db
|
||||
from as_platform.db.models import User
|
||||
from as_platform.fleet import service as fleet_svc
|
||||
from as_platform.fleet.mock_seed import reseed_demo_fleet, seed_demo_fleet
|
||||
|
||||
router = APIRouter(tags=["fleet"])
|
||||
|
||||
|
||||
class VehicleBody(BaseModel):
|
||||
plate_no: str
|
||||
tbox_device_id: str
|
||||
name: str | None = None
|
||||
team: str | None = None
|
||||
status: str | None = "active"
|
||||
|
||||
|
||||
class VehiclePatchBody(BaseModel):
|
||||
plate_no: str | None = None
|
||||
tbox_device_id: str | None = None
|
||||
name: str | None = None
|
||||
team: str | None = None
|
||||
status: str | None = None
|
||||
|
||||
|
||||
class TboxGpsBody(BaseModel):
|
||||
device_id: str
|
||||
lat: float
|
||||
lng: float
|
||||
ts: str | None = None
|
||||
speed_kmh: float | None = None
|
||||
heading: float | None = None
|
||||
plate_no: str | None = None
|
||||
run_signal: str | None = "active"
|
||||
engineer: str | None = None
|
||||
project: str | None = None
|
||||
batch: str | None = None
|
||||
|
||||
|
||||
class TboxGpsBatchBody(BaseModel):
|
||||
points: list[TboxGpsBody] = Field(..., min_length=1, max_length=100)
|
||||
|
||||
|
||||
def _check_fleet_enabled() -> None:
|
||||
if not FLEET_MAP_ENABLED:
|
||||
raise HTTPException(503, "车队地图功能未启用 (AS_FLEET_MAP_ENABLED)")
|
||||
|
||||
|
||||
def _verify_tbox_token(x_tbox_token: str | None = None, authorization: str | None = None) -> None:
|
||||
token = (x_tbox_token or "").strip()
|
||||
if not token and authorization:
|
||||
parts = authorization.split(None, 1)
|
||||
if len(parts) == 2 and parts[0].lower() == "bearer":
|
||||
token = parts[1].strip()
|
||||
if not TBOX_INGEST_TOKEN:
|
||||
return
|
||||
if token != TBOX_INGEST_TOKEN:
|
||||
raise HTTPException(401, "T-Box Token 无效")
|
||||
|
||||
|
||||
@router.get("/api/v1/fleet/map-config")
|
||||
def fleet_map_config(_user: Annotated[User, Depends(require_permission("read:fleet"))]) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
tile_provider = MAP_TILE_PROVIDER if MAP_TILE_PROVIDER in ("gaode", "osm") else "gaode"
|
||||
return {
|
||||
"enabled": True,
|
||||
"provider": tile_provider,
|
||||
"tileProvider": tile_provider,
|
||||
"amapKey": AMAP_KEY or "",
|
||||
"pollIntervalSec": 5,
|
||||
"note": "GPS 为 WGS84,高德底图为 GCJ-02,演示轨迹可能有轻微偏移",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/v1/fleet/live")
|
||||
def fleet_live(_user: Annotated[User, Depends(require_permission("read:fleet"))], db: Annotated[Session, Depends(get_db)]) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
return fleet_svc.get_live_fleet(db)
|
||||
|
||||
|
||||
@router.get("/api/v1/fleet/vehicles")
|
||||
def fleet_vehicles(_user: Annotated[User, Depends(require_permission("read:fleet"))], db: Annotated[Session, Depends(get_db)]) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
return {"items": fleet_svc.list_vehicles(db)}
|
||||
|
||||
|
||||
@router.get("/api/v1/fleet/vehicles/{vehicle_id}")
|
||||
def fleet_vehicle_get(
|
||||
vehicle_id: int,
|
||||
_user: Annotated[User, Depends(require_permission("read:fleet"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
item = fleet_svc.get_vehicle(db, vehicle_id)
|
||||
if not item:
|
||||
raise HTTPException(404, "车辆不存在")
|
||||
return item
|
||||
|
||||
|
||||
@router.post("/api/v1/fleet/vehicles")
|
||||
def fleet_vehicle_create(
|
||||
body: VehicleBody,
|
||||
_user: Annotated[User, Depends(require_permission("write:fleet"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
try:
|
||||
item = fleet_svc.create_vehicle(db, body.model_dump())
|
||||
db.commit()
|
||||
return {"ok": True, "vehicle": item}
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.patch("/api/v1/fleet/vehicles/{vehicle_id}")
|
||||
def fleet_vehicle_update(
|
||||
vehicle_id: int,
|
||||
body: VehiclePatchBody,
|
||||
_user: Annotated[User, Depends(require_permission("write:fleet"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
try:
|
||||
item = fleet_svc.update_vehicle(db, vehicle_id, body.model_dump(exclude_unset=True))
|
||||
db.commit()
|
||||
return {"ok": True, "vehicle": item}
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.delete("/api/v1/fleet/vehicles/{vehicle_id}")
|
||||
def fleet_vehicle_delete(
|
||||
vehicle_id: int,
|
||||
_user: Annotated[User, Depends(require_permission("write:fleet"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
try:
|
||||
fleet_svc.delete_vehicle(db, vehicle_id)
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.get("/api/v1/fleet/runs")
|
||||
def fleet_runs(
|
||||
_user: Annotated[User, Depends(require_permission("read:fleet"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
vehicle_id: int | None = None,
|
||||
status: str | None = None,
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
return fleet_svc.list_runs(db, vehicle_id=vehicle_id, status=status, offset=offset, limit=limit)
|
||||
|
||||
|
||||
@router.get("/api/v1/fleet/runs/{run_id}")
|
||||
def fleet_run_detail(run_id: int, _user: Annotated[User, Depends(require_permission("read:fleet"))], db: Annotated[Session, Depends(get_db)]) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
detail = fleet_svc.get_run_detail(db, run_id)
|
||||
if not detail:
|
||||
raise HTTPException(404, "趟次不存在")
|
||||
return detail
|
||||
|
||||
|
||||
@router.get("/api/v1/fleet/runs/{run_id}/track")
|
||||
def fleet_run_track(run_id: int, _user: Annotated[User, Depends(require_permission("read:fleet"))], db: Annotated[Session, Depends(get_db)]) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
return fleet_svc.get_run_track_geojson(db, run_id)
|
||||
|
||||
|
||||
@router.get("/api/v1/fleet/summary")
|
||||
def fleet_summary(_user: Annotated[User, Depends(require_permission("read:fleet"))], db: Annotated[Session, Depends(get_db)]) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
return fleet_svc.fleet_summary(db)
|
||||
|
||||
|
||||
@router.post("/api/v1/fleet/mock/seed")
|
||||
def fleet_mock_seed(
|
||||
_user: Annotated[User, Depends(require_permission("write:fleet"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
created = seed_demo_fleet(db)
|
||||
db.commit()
|
||||
return {"ok": True, "created": created}
|
||||
|
||||
|
||||
@router.post("/api/v1/fleet/mock/reseed")
|
||||
def fleet_mock_reseed(
|
||||
_user: Annotated[User, Depends(require_permission("write:fleet"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict[str, Any]:
|
||||
"""清空车队演示数据并重新注入(切换区域后使用)。"""
|
||||
_check_fleet_enabled()
|
||||
created = reseed_demo_fleet(db)
|
||||
db.commit()
|
||||
return {"ok": True, "created": created, "region": "changsha"}
|
||||
|
||||
|
||||
@router.post("/api/v1/fleet/mock/tick")
|
||||
def fleet_mock_tick(
|
||||
_user: Annotated[User, Depends(require_permission("write:fleet"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
n = fleet_svc.simulate_tick(db)
|
||||
db.commit()
|
||||
return {"ok": True, "advanced": n}
|
||||
|
||||
|
||||
@router.post("/api/v1/fleet/runs/import-gpx")
|
||||
async def fleet_import_gpx(
|
||||
_user: Annotated[User, Depends(require_permission("write:fleet"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
vehicle_id: int = Form(...),
|
||||
file: UploadFile = File(...),
|
||||
note: str | None = Form(None),
|
||||
) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
raw = await file.read()
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
text = raw.decode("utf-8", errors="ignore")
|
||||
try:
|
||||
result = fleet_svc.import_gpx_run(db, vehicle_id=vehicle_id, gpx_content=text, note=note)
|
||||
db.commit()
|
||||
return result
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.get("/api/v1/fleet/stream")
|
||||
def fleet_stream(
|
||||
_user: Annotated[User, Depends(require_permission("read:fleet"))],
|
||||
) -> StreamingResponse:
|
||||
"""SSE:推送 fleet.gps 事件(需 Redis)。"""
|
||||
_check_fleet_enabled()
|
||||
from as_platform.redis.bus import get_redis
|
||||
|
||||
def event_generator():
|
||||
r = get_redis()
|
||||
if not r:
|
||||
yield f"data: {json.dumps({'event': 'error', 'message': 'redis unavailable'}, ensure_ascii=False)}\n\n"
|
||||
return
|
||||
pubsub = r.pubsub()
|
||||
pubsub.subscribe("as:events")
|
||||
yield f"data: {json.dumps({'event': 'fleet.connected'}, ensure_ascii=False)}\n\n"
|
||||
while True:
|
||||
msg = pubsub.get_message(timeout=25)
|
||||
if msg is None:
|
||||
yield ": ping\n\n"
|
||||
continue
|
||||
if msg.get("type") != "message":
|
||||
continue
|
||||
raw = msg.get("data")
|
||||
if isinstance(raw, bytes):
|
||||
raw = raw.decode("utf-8", errors="ignore")
|
||||
if isinstance(raw, str) and "fleet." in raw:
|
||||
yield f"data: {raw}\n\n"
|
||||
|
||||
return StreamingResponse(event_generator(), media_type="text/event-stream")
|
||||
|
||||
|
||||
@router.post("/api/v1/tbox/gps")
|
||||
def tbox_gps(
|
||||
body: TboxGpsBody,
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
x_tbox_token: Annotated[str | None, Header(alias="X-Tbox-Token")] = None,
|
||||
authorization: Annotated[str | None, Header()] = None,
|
||||
) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
_verify_tbox_token(x_tbox_token, authorization)
|
||||
try:
|
||||
fleet_svc.close_idle_runs(db)
|
||||
result = fleet_svc.ingest_tbox_gps(db, body.model_dump())
|
||||
db.commit()
|
||||
return result
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.post("/api/v1/tbox/gps/batch")
|
||||
def tbox_gps_batch(
|
||||
body: TboxGpsBatchBody,
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
x_tbox_token: Annotated[str | None, Header(alias="X-Tbox-Token")] = None,
|
||||
authorization: Annotated[str | None, Header()] = None,
|
||||
) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
_verify_tbox_token(x_tbox_token, authorization)
|
||||
try:
|
||||
fleet_svc.close_idle_runs(db)
|
||||
result = fleet_svc.ingest_tbox_gps_batch(db, [p.model_dump() for p in body.points])
|
||||
db.commit()
|
||||
return result
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.post("/api/v1/fleet/runs/import-csv")
|
||||
async def fleet_import_csv(
|
||||
_user: Annotated[User, Depends(require_permission("write:fleet"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
vehicle_id: int = Form(...),
|
||||
file: UploadFile = File(...),
|
||||
note: str | None = Form(None),
|
||||
project: str | None = Form(None),
|
||||
batch: str | None = Form(None),
|
||||
) -> dict[str, Any]:
|
||||
_check_fleet_enabled()
|
||||
raw = await file.read()
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
text = raw.decode("utf-8", errors="ignore")
|
||||
try:
|
||||
result = fleet_svc.import_csv_run(
|
||||
db,
|
||||
vehicle_id=vehicle_id,
|
||||
csv_content=text,
|
||||
note=note,
|
||||
project=project,
|
||||
batch=batch,
|
||||
)
|
||||
db.commit()
|
||||
return result
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
445
platform/as_platform/api/labeling_routes.py
Normal file
445
platform/as_platform/api/labeling_routes.py
Normal file
@@ -0,0 +1,445 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import zipfile
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from as_platform.auth.deps import require_any_permission, require_permission
|
||||
from as_platform.db.models import User
|
||||
from as_platform.labeling.annotate import (
|
||||
campaign_bootstrap,
|
||||
campaign_tasks,
|
||||
get_annotation,
|
||||
resolve_media_file,
|
||||
save_annotation,
|
||||
)
|
||||
from as_platform.labeling.lock import acquire_lock, release_lock, renew_lock
|
||||
from as_platform.labeling.progress import (
|
||||
assign_tasks_even,
|
||||
assign_tasks_explicit,
|
||||
campaign_progress,
|
||||
release_task_assignment,
|
||||
reassign_task,
|
||||
user_is_coordinator,
|
||||
)
|
||||
from as_platform.labeling.service import (
|
||||
assign_campaign,
|
||||
get_campaign,
|
||||
list_campaign_export_jobs,
|
||||
list_labeling_assignees,
|
||||
list_labeling_batches,
|
||||
open_campaign,
|
||||
submit_campaign,
|
||||
trigger_labeling_export,
|
||||
)
|
||||
from as_platform.labeling.vendor_import import import_vendor_zip, list_registry_profiles
|
||||
|
||||
router = APIRouter(tags=["labeling"])
|
||||
|
||||
|
||||
class OpenCampaignBody(BaseModel):
|
||||
project: str = Field(..., pattern="^(dms|lane)$")
|
||||
task: str
|
||||
batch: str
|
||||
mode: str | None = None
|
||||
pack: str | None = None
|
||||
location: str = "inbox"
|
||||
|
||||
|
||||
class AssignCampaignBody(BaseModel):
|
||||
user_id: int | None = None
|
||||
|
||||
|
||||
class AnnotationBody(BaseModel):
|
||||
result: list[dict[str, Any]] | dict[str, Any] | None = None
|
||||
annotations: list[dict[str, Any]] | None = None
|
||||
|
||||
|
||||
class AssignTasksExplicitItem(BaseModel):
|
||||
user_id: int
|
||||
task_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AssignTasksBody(BaseModel):
|
||||
mode: str = "even"
|
||||
user_ids: list[int] | None = None
|
||||
items: list[AssignTasksExplicitItem] | None = None
|
||||
|
||||
|
||||
class ReassignTaskBody(BaseModel):
|
||||
user_id: int
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/assignees")
|
||||
def api_labeling_assignees(
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
return list_labeling_assignees()
|
||||
|
||||
|
||||
@router.patch("/api/v1/labeling/campaigns/{campaign_id}/assign")
|
||||
def api_assign_campaign(
|
||||
campaign_id: str,
|
||||
body: AssignCampaignBody,
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return assign_campaign(campaign_id, body.user_id)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "campaign not found") from None
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/batches")
|
||||
def api_labeling_batches(
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
stage: str | None = Query(None),
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
return list_labeling_batches(stage=stage, offset=offset, limit=limit)
|
||||
|
||||
|
||||
@router.post("/api/v1/labeling/campaigns/open")
|
||||
def api_open_campaign(
|
||||
body: OpenCampaignBody,
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
return open_campaign(
|
||||
project=body.project,
|
||||
task=body.task,
|
||||
batch=body.batch,
|
||||
mode=body.mode,
|
||||
pack=body.pack,
|
||||
location=body.location,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/campaigns/{campaign_id}")
|
||||
def api_get_campaign(
|
||||
campaign_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
row = get_campaign(campaign_id)
|
||||
if not row:
|
||||
raise HTTPException(404, "campaign not found")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/campaigns/{campaign_id}/bootstrap")
|
||||
def api_campaign_bootstrap(
|
||||
campaign_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return campaign_bootstrap(campaign_id)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "campaign not found") from None
|
||||
except Exception as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/campaigns/{campaign_id}/progress")
|
||||
def api_campaign_progress(
|
||||
campaign_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return campaign_progress(campaign_id)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "campaign not found") from None
|
||||
|
||||
|
||||
@router.post("/api/v1/labeling/campaigns/{campaign_id}/assign-tasks")
|
||||
def api_assign_tasks(
|
||||
campaign_id: str,
|
||||
body: AssignTasksBody,
|
||||
user: Annotated[User, Depends(require_any_permission("write:labeling_assign", "read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
if not user_is_coordinator(user):
|
||||
raise HTTPException(403, "仅协调员可分配任务")
|
||||
try:
|
||||
if body.mode == "explicit":
|
||||
items = [{"user_id": i.user_id, "task_ids": i.task_ids} for i in (body.items or [])]
|
||||
return assign_tasks_explicit(campaign_id, items, assigned_by_user_id=user.id)
|
||||
if not body.user_ids:
|
||||
raise ValueError("even 模式需要 user_ids")
|
||||
return assign_tasks_even(campaign_id, body.user_ids, assigned_by_user_id=user.id)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "campaign not found") from None
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.patch("/api/v1/labeling/campaigns/{campaign_id}/assignments/{task_id}")
|
||||
def api_reassign_task(
|
||||
campaign_id: str,
|
||||
task_id: str,
|
||||
body: ReassignTaskBody,
|
||||
user: Annotated[User, Depends(require_any_permission("write:labeling_assign", "read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
if not user_is_coordinator(user):
|
||||
raise HTTPException(403, "仅协调员可改派任务")
|
||||
try:
|
||||
return reassign_task(campaign_id, task_id, body.user_id)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "assignment not found") from None
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.delete("/api/v1/labeling/campaigns/{campaign_id}/assignments/{task_id}")
|
||||
def api_release_assignment(
|
||||
campaign_id: str,
|
||||
task_id: str,
|
||||
user: Annotated[User, Depends(require_any_permission("write:labeling_assign", "read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
if not user_is_coordinator(user):
|
||||
raise HTTPException(403, "仅协调员可释放任务")
|
||||
try:
|
||||
return release_task_assignment(campaign_id, task_id)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "assignment not found") from None
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/campaigns/{campaign_id}/tasks")
|
||||
def api_campaign_tasks(
|
||||
campaign_id: str,
|
||||
user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(50, ge=1, le=200),
|
||||
assignee: str | None = Query(None),
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
eff = assignee
|
||||
if eff is None and not user_is_coordinator(user):
|
||||
eff = "me"
|
||||
return campaign_tasks(campaign_id, offset=offset, limit=limit, user=user, assignee=eff)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "campaign not found") from None
|
||||
except PermissionError as e:
|
||||
raise HTTPException(403, str(e)) from e
|
||||
except Exception as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/media/{campaign_id}/{file_path:path}")
|
||||
def api_labeling_media(
|
||||
campaign_id: str,
|
||||
file_path: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
):
|
||||
try:
|
||||
target = resolve_media_file(campaign_id, file_path)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "not found") from None
|
||||
except PermissionError:
|
||||
raise HTTPException(403, "forbidden") from None
|
||||
except Exception as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
return FileResponse(target)
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/campaigns/{campaign_id}/annotations/{task_id}")
|
||||
def api_get_annotation(
|
||||
campaign_id: str,
|
||||
task_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return get_annotation(campaign_id, task_id)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "campaign not found") from None
|
||||
|
||||
|
||||
@router.put("/api/v1/labeling/campaigns/{campaign_id}/annotations/{task_id}")
|
||||
def api_put_annotation(
|
||||
campaign_id: str,
|
||||
task_id: str,
|
||||
body: AnnotationBody,
|
||||
user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return save_annotation(campaign_id, task_id, body.model_dump(exclude_none=True), user=user)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "campaign not found") from None
|
||||
except PermissionError as e:
|
||||
raise HTTPException(403, str(e)) from e
|
||||
except Exception as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.post("/api/v1/labeling/campaigns/{campaign_id}/export")
|
||||
def api_labeling_export(
|
||||
campaign_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return trigger_labeling_export(campaign_id)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "campaign not found") from None
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/campaigns/{campaign_id}/export-jobs")
|
||||
def api_campaign_export_jobs(
|
||||
campaign_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
limit: int = Query(30, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
if not get_campaign(campaign_id):
|
||||
raise HTTPException(404, "campaign not found")
|
||||
return list_campaign_export_jobs(campaign_id, limit=limit)
|
||||
|
||||
|
||||
@router.post("/api/v1/labeling/campaigns/{campaign_id}/submit")
|
||||
def api_campaign_submit(
|
||||
campaign_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return submit_campaign(campaign_id)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from None
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "campaign not found") from None
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/registry-profiles")
|
||||
def api_labeling_registry_profiles(
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
return list_registry_profiles()
|
||||
|
||||
|
||||
@router.post("/api/v1/labeling/campaigns/{campaign_id}/import-vendor")
|
||||
async def api_import_vendor(
|
||||
campaign_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("write:labeling_vendor"))],
|
||||
file: UploadFile = File(...),
|
||||
) -> dict[str, Any]:
|
||||
raw = await file.read()
|
||||
try:
|
||||
return import_vendor_zip(campaign_id, raw)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(404, "campaign not found") from None
|
||||
except (zipfile.BadZipFile, ValueError) as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.post("/api/v1/labeling/campaigns/{campaign_id}/tasks/{task_id}/lock")
|
||||
def api_labeling_lock_acquire(
|
||||
campaign_id: str,
|
||||
task_id: str,
|
||||
user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
if not get_campaign(campaign_id):
|
||||
raise HTTPException(404, "campaign not found")
|
||||
result = acquire_lock(campaign_id, task_id, user_id=user.id, user_name=user.name)
|
||||
if not result.get("ok"):
|
||||
raise HTTPException(409, detail=result)
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/api/v1/labeling/campaigns/{campaign_id}/tasks/{task_id}/lock")
|
||||
def api_labeling_lock_release(
|
||||
campaign_id: str,
|
||||
task_id: str,
|
||||
user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
result = release_lock(campaign_id, task_id, user_id=user.id)
|
||||
if not result.get("ok"):
|
||||
raise HTTPException(409, detail=result)
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/v1/labeling/campaigns/{campaign_id}/tasks/{task_id}/lock/renew")
|
||||
def api_labeling_lock_renew(
|
||||
campaign_id: str,
|
||||
task_id: str,
|
||||
user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, Any]:
|
||||
result = renew_lock(campaign_id, task_id, user_id=user.id)
|
||||
if not result.get("ok"):
|
||||
raise HTTPException(409, detail=result)
|
||||
return result
|
||||
|
||||
|
||||
# ── 标注质检 (Quality Review) ──
|
||||
|
||||
class ReviewScoreBody(BaseModel):
|
||||
image_path: str
|
||||
score: str # good / fine / bad
|
||||
comment: str | None = None
|
||||
|
||||
|
||||
class ReviewBatchBody(BaseModel):
|
||||
scores: list[ReviewScoreBody]
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/campaigns/{campaign_id}/review-queue")
|
||||
def api_review_queue(
|
||||
campaign_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("write:approval_review"))],
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
from as_platform.audit.review import get_review_queue
|
||||
return get_review_queue(campaign_id, offset=offset, limit=limit)
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/campaigns/{campaign_id}/review-image")
|
||||
def api_review_image(
|
||||
campaign_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("write:approval_review"))],
|
||||
path: str = Query(...),
|
||||
) -> FileResponse:
|
||||
from as_platform.audit.review import get_review_image
|
||||
import tempfile
|
||||
try:
|
||||
# Get class names from registry
|
||||
import yaml
|
||||
from as_platform.data.core import load_wf, proj_root
|
||||
wf = load_wf()
|
||||
root = proj_root(wf, "dms")
|
||||
reg = yaml.safe_load((root / wf["projects"]["dms"]["registry"]).read_text())
|
||||
# Build class_names dict from campaign scope
|
||||
class_names: dict[int, str] = {}
|
||||
# Try to get from the specific task
|
||||
tasks = reg.get("tasks", {})
|
||||
for task_cfg in tasks.values():
|
||||
names = task_cfg.get("names")
|
||||
if isinstance(names, list):
|
||||
for i, n in enumerate(names):
|
||||
class_names[i] = n
|
||||
|
||||
data = get_review_image(campaign_id, path, class_names)
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=".jpg", delete=False)
|
||||
tmp.write(data)
|
||||
tmp.close()
|
||||
return FileResponse(tmp.name, media_type="image/jpeg")
|
||||
except FileNotFoundError as e:
|
||||
raise HTTPException(404, str(e)) from e
|
||||
|
||||
|
||||
@router.post("/api/v1/labeling/campaigns/{campaign_id}/review-submit")
|
||||
def api_review_submit(
|
||||
campaign_id: str,
|
||||
body: ReviewBatchBody,
|
||||
user: Annotated[User, Depends(require_permission("write:approval_review"))],
|
||||
) -> dict[str, Any]:
|
||||
from as_platform.audit.review import submit_review_scores
|
||||
items = [s.model_dump() for s in body.scores]
|
||||
return submit_review_scores(campaign_id, items, reviewer_user_id=user.id, reviewer_name=user.name)
|
||||
|
||||
|
||||
@router.get("/api/v1/labeling/campaigns/{campaign_id}/review-progress")
|
||||
def api_review_progress(
|
||||
campaign_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:pending"))],
|
||||
) -> dict[str, int]:
|
||||
from as_platform.audit.review import review_progress
|
||||
return review_progress(campaign_id)
|
||||
155
platform/as_platform/api/models_routes.py
Normal file
155
platform/as_platform/api/models_routes.py
Normal file
@@ -0,0 +1,155 @@
|
||||
"""模型管理 API 路由 — 模块化前缀 /api/v1/models
|
||||
|
||||
本路由将 training/* 功能以模块化路径重新暴露,同时保留旧路径兼容。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from as_platform.auth.deps import can_submit_action, get_current_user, require_permission
|
||||
from as_platform.db.models import User
|
||||
from as_platform.data.versions import create_snapshot, diff_versions, get_version, list_versions
|
||||
from as_platform.training.service import (
|
||||
TRAINING_ACTIONS,
|
||||
create_training_submission,
|
||||
get_model_registry,
|
||||
get_training_record,
|
||||
list_training_records,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/v1/models", tags=["models"])
|
||||
|
||||
|
||||
class CreateTrainingBody(BaseModel):
|
||||
action: str
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
note: str | None = None
|
||||
|
||||
|
||||
@router.get("/actions")
|
||||
def api_models_actions(_user: Annotated[User, Depends(require_permission("read:jobs"))]) -> dict[str, Any]:
|
||||
"""可用的训练/评估/晋级操作列表"""
|
||||
from as_platform.audit.queue import ACTION_LABELS
|
||||
|
||||
return {
|
||||
"actions": [
|
||||
{"id": action, "label": ACTION_LABELS.get(action, action)}
|
||||
for action in sorted(TRAINING_ACTIONS)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@router.get("/registry")
|
||||
def api_models_registry(
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
project: str = Query("dms"),
|
||||
task: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""模型注册表"""
|
||||
return get_model_registry(project=project, task=task)
|
||||
|
||||
|
||||
@router.get("/records")
|
||||
def api_models_records(
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
project: str | None = None,
|
||||
kind: str | None = None,
|
||||
status: str | None = None,
|
||||
task: str | None = None,
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
"""训练/评估记录列表"""
|
||||
return list_training_records(
|
||||
project=project, kind=kind, status=status, task=task, offset=offset, limit=limit
|
||||
)
|
||||
|
||||
|
||||
@router.get("/records/{record_id}")
|
||||
def api_models_record(
|
||||
record_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
) -> dict[str, Any]:
|
||||
"""单条训练记录详情"""
|
||||
rec = get_training_record(record_id)
|
||||
if not rec:
|
||||
raise HTTPException(404, "训练记录不存在")
|
||||
return rec
|
||||
|
||||
|
||||
@router.post("/records")
|
||||
def api_models_create_record(
|
||||
body: CreateTrainingBody,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
) -> dict[str, Any]:
|
||||
"""提交训练/评估/晋级申请"""
|
||||
if not can_submit_action(user, body.action):
|
||||
raise HTTPException(403, f"无权提交: {body.action}")
|
||||
try:
|
||||
return create_training_submission(
|
||||
body.action,
|
||||
body.params,
|
||||
submitted_by=user.name,
|
||||
submitted_by_user_id=user.id,
|
||||
note=body.note,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
# ── 数据集版本管理 ──
|
||||
|
||||
class CreateSnapshotBody(BaseModel):
|
||||
project: str = "dms"
|
||||
description: str = ""
|
||||
|
||||
|
||||
@router.get("/datasets")
|
||||
def api_list_dataset_versions(
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
project: str = Query("dms"),
|
||||
) -> dict[str, Any]:
|
||||
return {"items": list_versions(project)}
|
||||
|
||||
|
||||
@router.post("/datasets/snapshot")
|
||||
def api_create_snapshot(
|
||||
body: CreateSnapshotBody,
|
||||
user: Annotated[User, Depends(require_permission("write:approval_submit"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
result = create_snapshot(body.project, body.description, author=user.name)
|
||||
from as_platform.audit.log_utils import log_op
|
||||
log_op(user_id=user.id, user_name=user.name, category="data", action="create_snapshot",
|
||||
target_type="snapshot", target_id=result.get("version_id"), summary=f"创建快照: {result.get('version_id')} ({body.project})")
|
||||
return result
|
||||
except Exception as e:
|
||||
raise HTTPException(500, str(e)) from e
|
||||
|
||||
|
||||
@router.get("/datasets/{version_id}")
|
||||
def api_get_version(
|
||||
version_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
project: str = Query("dms"),
|
||||
) -> dict[str, Any]:
|
||||
v = get_version(project, version_id)
|
||||
if not v:
|
||||
raise HTTPException(404, f"版本 {version_id} 不存在")
|
||||
return v
|
||||
|
||||
|
||||
@router.get("/datasets/{version_id}/diff")
|
||||
def api_diff_versions(
|
||||
version_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
compare: str = Query(...),
|
||||
project: str = Query("dms"),
|
||||
) -> dict[str, Any]:
|
||||
result = diff_versions(project, compare, version_id)
|
||||
if "error" in result:
|
||||
raise HTTPException(400, result["error"])
|
||||
return result
|
||||
@@ -23,6 +23,12 @@ from as_platform.agents.graphs.train_promote_flow import run_train_promote_flow
|
||||
from as_platform.agents.tools import TOOL_REGISTRY, invoke_tool
|
||||
from as_platform.agents.trace import get_trace, list_traces
|
||||
from as_platform.api.auth_routes import router as auth_router
|
||||
from as_platform.api.fleet_routes import router as fleet_router
|
||||
from as_platform.api.delivery_routes import router as delivery_router
|
||||
from as_platform.api.feishu_routes import router as feishu_router
|
||||
from as_platform.api.labeling_routes import router as labeling_router
|
||||
from as_platform.api.models_routes import router as models_router
|
||||
from as_platform.api.system_routes import router as system_router
|
||||
from as_platform.audit.queue import (
|
||||
ACTION_LABELS,
|
||||
ACTIONS_REQUIRING_APPROVAL,
|
||||
@@ -34,7 +40,16 @@ from as_platform.audit.queue import (
|
||||
)
|
||||
from as_platform.audit.preview import find_image_ref, list_scope_images, render_overlay, resolve_approval_scope
|
||||
from as_platform.auth.deps import can_submit_action, get_current_user, require_any_permission, require_permission
|
||||
from as_platform.config import IS_POSTGRES, PLATFORM_DIR, PLATFORM_WEB, WORKSPACE
|
||||
from as_platform.config import (
|
||||
FEISHU_BITABLE_SYNC_ENABLED,
|
||||
FEISHU_BITABLE_SYNC_INTERVAL_SEC,
|
||||
FLEET_MOCK_SIMULATE,
|
||||
FLEET_SIM_INTERVAL_SEC,
|
||||
IS_POSTGRES,
|
||||
PLATFORM_DIR,
|
||||
PLATFORM_WEB,
|
||||
WORKSPACE,
|
||||
)
|
||||
from as_platform.data.core import get_catalog, get_pending_report, register_batch, warmup_catalog_cache
|
||||
from as_platform.data.ingest import UnknownFormatError, inspect_uploaded_dataset
|
||||
from as_platform.data.lake import (
|
||||
@@ -42,10 +57,11 @@ from as_platform.data.lake import (
|
||||
get_candidate,
|
||||
link_candidate_analysis_job,
|
||||
list_candidates as list_data_candidates,
|
||||
promote_candidate_to_inbox,
|
||||
write_candidate_upload,
|
||||
)
|
||||
from as_platform.data.organize import organize_batch
|
||||
from as_platform.db.engine import check_connection
|
||||
from as_platform.db.engine import check_connection, session_scope
|
||||
from as_platform.db.init_db import init_database
|
||||
from as_platform.db.models import User
|
||||
from as_platform.jobs.queue import enqueue_job, get_job, list_jobs
|
||||
@@ -59,16 +75,51 @@ from as_platform.training.service import (
|
||||
)
|
||||
|
||||
|
||||
def _feishu_bitable_sync_loop() -> None:
|
||||
import time
|
||||
|
||||
from as_platform.jobs.feishu_bitable_sync import run_sync_cycle
|
||||
|
||||
while True:
|
||||
time.sleep(FEISHU_BITABLE_SYNC_INTERVAL_SEC)
|
||||
try:
|
||||
run_sync_cycle()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _fleet_sim_loop() -> None:
|
||||
import time
|
||||
from as_platform.fleet import service as fleet_svc
|
||||
while True:
|
||||
time.sleep(max(3, FLEET_SIM_INTERVAL_SEC))
|
||||
try:
|
||||
with session_scope() as db:
|
||||
fleet_svc.simulate_tick(db)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
init_database()
|
||||
threading.Thread(target=warmup_catalog_cache, daemon=True, name="catalog-warmup").start()
|
||||
if FLEET_MOCK_SIMULATE:
|
||||
threading.Thread(target=_fleet_sim_loop, daemon=True, name="fleet-sim",).start()
|
||||
if FEISHU_BITABLE_SYNC_ENABLED:
|
||||
threading.Thread(target=_feishu_bitable_sync_loop, daemon=True, name="feishu-bitable-sync").start()
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(title="华胥智能主动安全平台", version="1.1.0", lifespan=lifespan)
|
||||
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"])
|
||||
app.include_router(auth_router)
|
||||
app.include_router(fleet_router)
|
||||
app.include_router(labeling_router)
|
||||
app.include_router(feishu_router)
|
||||
app.include_router(delivery_router)
|
||||
app.include_router(models_router) # 模型管理: /api/v1/models/*
|
||||
app.include_router(system_router) # 系统管理: /api/v1/system/*
|
||||
_UI_DIR: Path | None = None
|
||||
|
||||
|
||||
@@ -152,6 +203,19 @@ def api_pending(_user: Annotated[User, Depends(require_permission("read:pending"
|
||||
return get_pending_report()
|
||||
|
||||
|
||||
@app.get("/api/v1/pending/gates")
|
||||
def api_pending_gates(_user: Annotated[User, Depends(require_permission("read:pending"))]) -> dict[str, Any]:
|
||||
"""ML 自动化 P0:build 门禁与 manifest 对齐说明。"""
|
||||
return {
|
||||
"build_validate": (
|
||||
"python as.py build dms <task> <pack> 入库后默认执行 scripts/validate_dms_tasks.py;"
|
||||
"仅调试可用 --skip-validate"
|
||||
),
|
||||
"manifest_smoke": "bash HSAP/scripts/smoke_manifest_alignment.sh",
|
||||
"pending_cli": "python as.py pending",
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/v1/catalog")
|
||||
def api_catalog(
|
||||
_user: Annotated[User, Depends(require_permission("read:catalog"))],
|
||||
@@ -193,9 +257,10 @@ def api_actions(_user: Annotated[User, Depends(require_permission("read:audit"))
|
||||
def api_jobs(
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
status: str | None = None,
|
||||
limit: int = Query(100, le=500),
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
return {"items": list_jobs(status=status, limit=limit)}
|
||||
return list_jobs(status=status, offset=offset, limit=limit)
|
||||
|
||||
|
||||
@app.get("/api/v1/jobs/{job_id}")
|
||||
@@ -223,9 +288,12 @@ def api_training_records(
|
||||
kind: str | None = None,
|
||||
status: str | None = None,
|
||||
task: str | None = None,
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
return list_training_records(project=project, kind=kind, status=status, task=task, limit=limit)
|
||||
return list_training_records(
|
||||
project=project, kind=kind, status=status, task=task, offset=offset, limit=limit
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/training/records/{job_id}")
|
||||
@@ -310,9 +378,10 @@ def api_agent_invoke(body: AgentInvokeBody, _user: Annotated[User, Depends(get_c
|
||||
def api_list_approvals(
|
||||
_user: Annotated[User, Depends(require_permission("read:audit"))],
|
||||
status: str | None = None,
|
||||
limit: int = Query(100, le=500),
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
return {"items": list_approvals(status=status, limit=limit)}
|
||||
return list_approvals(status=status, offset=offset, limit=limit)
|
||||
|
||||
|
||||
@app.get("/api/v1/approvals/{record_id}")
|
||||
@@ -346,13 +415,29 @@ def api_approval_preview(
|
||||
"exists": batch_dir.is_dir(),
|
||||
}
|
||||
)
|
||||
params = rec.get("params") or {}
|
||||
title = str(rec.get("action_label") or rec.get("action") or record_id)
|
||||
summary = ""
|
||||
if rec.get("action") == "delivery_ingest":
|
||||
title = f"数据送标入湖 · {params.get('batch_name') or '—'}"
|
||||
parts = [
|
||||
f"项目 {params.get('project') or 'dms'}",
|
||||
f"任务 {params.get('task') or '—'}",
|
||||
f"路径 {params.get('data_path') or '—'}",
|
||||
]
|
||||
if params.get("estimated_count"):
|
||||
parts.append(f"约 {params['estimated_count']} 张")
|
||||
summary = " · ".join(parts)
|
||||
return {
|
||||
"approval": rec,
|
||||
"title": title,
|
||||
"summary": summary,
|
||||
"scope_label": scope.get("scope_label"),
|
||||
"task": scope.get("task"),
|
||||
"pack": scope.get("pack"),
|
||||
"class_names": scope.get("class_names"),
|
||||
"batches": batch_summaries,
|
||||
"delivery": params if rec.get("action") == "delivery_ingest" else None,
|
||||
}
|
||||
|
||||
|
||||
@@ -456,22 +541,29 @@ def api_reject(record_id: str, body: ReviewBody, user: Annotated[User, Depends(r
|
||||
|
||||
@app.post("/api/v1/register-batch")
|
||||
def api_register_batch(body: RegisterBatchBody, user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
|
||||
result = None
|
||||
if body.skip_audit:
|
||||
if not can_submit_action(user, "register_batch"):
|
||||
raise HTTPException(403, "无权登记批次")
|
||||
try:
|
||||
return register_batch(None, body.project, body.task, body.batch, pack=body.pack, stage=body.stage, engineer=body.engineer, location=body.location)
|
||||
result = register_batch(None, body.project, body.task, body.batch, pack=body.pack, stage=body.stage, engineer=body.engineer, location=body.location)
|
||||
except (ValueError, FileNotFoundError) as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
if not can_submit_action(user, "register_batch"):
|
||||
raise HTTPException(403, "无权提交登记审核")
|
||||
return submit_approval(
|
||||
"register_batch",
|
||||
body.model_dump(exclude={"submitted_by", "skip_audit"}),
|
||||
submitted_by=user.name,
|
||||
submitted_by_user_id=user.id,
|
||||
note="登记 batch.meta",
|
||||
)
|
||||
else:
|
||||
if not can_submit_action(user, "register_batch"):
|
||||
raise HTTPException(403, "无权提交登记审核")
|
||||
result = submit_approval(
|
||||
"register_batch",
|
||||
body.model_dump(exclude={"submitted_by", "skip_audit"}),
|
||||
submitted_by=user.name,
|
||||
submitted_by_user_id=user.id,
|
||||
note="登记 batch.meta",
|
||||
)
|
||||
# 审计日志
|
||||
from as_platform.audit.log_utils import log_op
|
||||
log_op(user_id=user.id, user_name=user.name, category="data", action="register_batch",
|
||||
target_type="batch", target_id=f"{body.project}/{body.task}/{body.batch}", summary=f"登记批次: {body.project}/{body.task}/{body.batch} → {body.stage}")
|
||||
return result
|
||||
|
||||
|
||||
@app.post("/api/v1/data/organize")
|
||||
@@ -498,6 +590,7 @@ async def api_upload_file(
|
||||
project: Annotated[str, Form()],
|
||||
user: Annotated[User, Depends(require_any_permission("write:approval_submit", "write:approval_submit:register"))],
|
||||
task: Annotated[str | None, Form()] = None,
|
||||
mode: Annotated[str | None, Form()] = None,
|
||||
file: UploadFile = File(...),
|
||||
) -> dict[str, Any]:
|
||||
if not file.filename:
|
||||
@@ -506,6 +599,7 @@ async def api_upload_file(
|
||||
candidate = create_uploaded_candidate(
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
original_name=file.filename,
|
||||
upload_size_bytes=0,
|
||||
submitted_by_name=user.name if user else None,
|
||||
@@ -525,9 +619,10 @@ async def api_upload_file(
|
||||
@app.get("/api/v1/data/candidates")
|
||||
def api_data_candidates(
|
||||
_user: Annotated[User, Depends(require_permission("read:catalog"))],
|
||||
limit: int = Query(50, ge=1, le=500),
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
return {"items": list_data_candidates(limit=limit)}
|
||||
return list_data_candidates(offset=offset, limit=limit)
|
||||
|
||||
|
||||
@app.get("/api/v1/data/candidates/{candidate_id}")
|
||||
@@ -538,18 +633,272 @@ def api_data_candidate(candidate_id: str, _user: Annotated[User, Depends(require
|
||||
return item
|
||||
|
||||
|
||||
class PromoteInboxBody(BaseModel):
|
||||
batch: str | None = None
|
||||
mode: str | None = None
|
||||
|
||||
|
||||
@app.post("/api/v1/data/candidates/{candidate_id}/promote-inbox")
|
||||
def api_promote_candidate_inbox(
|
||||
candidate_id: str,
|
||||
body: PromoteInboxBody,
|
||||
_user: Annotated[User, Depends(require_any_permission("write:approval_submit", "write:approval_submit:register"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return promote_candidate_to_inbox(candidate_id, batch=body.batch, mode=body.mode)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
except FileNotFoundError as e:
|
||||
raise HTTPException(404, str(e)) from e
|
||||
|
||||
|
||||
@app.get("/api/v1/data/registry-tasks")
|
||||
def api_registry_tasks(
|
||||
_user: Annotated[User, Depends(require_permission("read:catalog"))],
|
||||
project: str = Query("dms"),
|
||||
) -> dict[str, Any]:
|
||||
import yaml
|
||||
|
||||
from as_platform.data.core import load_wf, proj_root
|
||||
|
||||
wf = load_wf()
|
||||
if project != "dms":
|
||||
return {"project": project, "tasks": {}}
|
||||
root = proj_root(wf, "dms")
|
||||
reg_path = root / wf["projects"]["dms"]["registry"]
|
||||
if not reg_path.is_file():
|
||||
return {"project": project, "tasks": {}}
|
||||
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
|
||||
import sys
|
||||
|
||||
scripts = WORKSPACE / "datasets" / "dms" / "scripts"
|
||||
if str(scripts) not in sys.path:
|
||||
sys.path.insert(0, str(scripts))
|
||||
from task_registry import task_defs_for_pending
|
||||
|
||||
return {"project": project, "tasks": task_defs_for_pending(reg)}
|
||||
|
||||
|
||||
@app.get("/api/v1/data/scan-inbox")
|
||||
def api_scan_inbox(
|
||||
_user: Annotated[User, Depends(require_permission("read:catalog"))],
|
||||
project: str = Query("dms"),
|
||||
) -> dict[str, Any]:
|
||||
"""扫描 inbox 目录,返回未登记的新批次。"""
|
||||
from as_platform.data.core import get_pending_report, load_wf, proj_root
|
||||
|
||||
wf = load_wf()
|
||||
root = proj_root(wf, project)
|
||||
inbox = root / "inbox"
|
||||
if not inbox.is_dir():
|
||||
return {"project": project, "items": [], "inbox_path": str(inbox)}
|
||||
|
||||
report = get_pending_report()
|
||||
registered = {b.get("batch", "") for b in report.get("batches", [])}
|
||||
|
||||
items: list[dict[str, Any]] = []
|
||||
for task_dir in sorted(inbox.iterdir()):
|
||||
if not task_dir.is_dir():
|
||||
continue
|
||||
for batch_dir in sorted(task_dir.iterdir()):
|
||||
if not batch_dir.is_dir():
|
||||
continue
|
||||
batch_name = batch_dir.name
|
||||
task_name = task_dir.name
|
||||
if batch_name in registered:
|
||||
continue # 已登记
|
||||
|
||||
# Count images
|
||||
img_count = 0
|
||||
lbl_count = 0
|
||||
has_labels = False
|
||||
for ext in ["*.jpg", "*.jpeg", "*.png", "*.bmp"]:
|
||||
for _ in batch_dir.glob(ext):
|
||||
img_count += 1
|
||||
if (batch_dir / "labels").is_dir():
|
||||
has_labels = True
|
||||
for ext in ["*.txt", "*.json"]:
|
||||
for _ in (batch_dir / "labels").glob(ext):
|
||||
lbl_count += 1
|
||||
|
||||
items.append({
|
||||
"project": project,
|
||||
"task": task_name,
|
||||
"batch": batch_name,
|
||||
"path": str(batch_dir.relative_to(root)),
|
||||
"images": img_count,
|
||||
"labels": lbl_count,
|
||||
"has_labels": has_labels,
|
||||
"stage_hint": "returned" if has_labels and lbl_count > 0 else "raw_pool",
|
||||
})
|
||||
|
||||
return {"project": project, "items": items, "inbox_path": str(inbox)}
|
||||
|
||||
|
||||
@app.get("/api/v1/dashboard")
|
||||
def api_dashboard(_user: Annotated[User, Depends(get_current_user)]) -> dict[str, Any]:
|
||||
"""首页仪表盘聚合数据。"""
|
||||
from as_platform.data.core import get_pending_report, get_catalog
|
||||
from as_platform.jobs.queue import list_jobs
|
||||
|
||||
# 批次统计
|
||||
report = get_pending_report()
|
||||
batches = report.get("batches", []) or []
|
||||
stage_counts: dict[str, int] = {"raw_pool": 0, "out_for_labeling": 0, "labeling_submitted": 0, "returned": 0, "ingested": 0}
|
||||
for b in batches:
|
||||
s = b.get("stage", "raw_pool") if isinstance(b, dict) else "raw_pool"
|
||||
stage_counts[s] = stage_counts.get(s, 0) + 1
|
||||
|
||||
# 审核统计
|
||||
approvals_pending = list_approvals(status="pending", limit=200)
|
||||
pending_approvals = approvals_pending.get("total", 0)
|
||||
|
||||
# Job 统计
|
||||
jobs_data = list_jobs(limit=5)
|
||||
jobs = jobs_data.get("items", []) or []
|
||||
running_jobs = len([j for j in jobs if isinstance(j, dict) and j.get("status") == "running"])
|
||||
|
||||
# 模型统计
|
||||
try:
|
||||
from as_platform.training.service import get_model_registry
|
||||
models = get_model_registry(project="dms")
|
||||
model_count = len((models.get("models") or []) if isinstance(models, dict) else [])
|
||||
except Exception:
|
||||
model_count = 0
|
||||
|
||||
# 训练记录
|
||||
try:
|
||||
from as_platform.training.service import list_training_records
|
||||
records = list_training_records(limit=5)
|
||||
recent_records = (records.get("items") or [])[:5] if isinstance(records, dict) else []
|
||||
except Exception:
|
||||
recent_records = []
|
||||
|
||||
# 车队
|
||||
try:
|
||||
from as_platform.fleet import service as fleet_svc
|
||||
from as_platform.db.engine import session_scope
|
||||
with session_scope() as db:
|
||||
summary = fleet_svc.get_summary(db) if hasattr(fleet_svc, "get_summary") else {}
|
||||
except Exception:
|
||||
summary = {}
|
||||
|
||||
# 最近活动
|
||||
activity: list[dict[str, Any]] = []
|
||||
for j in jobs[:5]:
|
||||
if isinstance(j, dict):
|
||||
activity.append({"type": "job", "id": j.get("id"), "action": j.get("action"), "status": j.get("status"), "time": j.get("created_at")})
|
||||
for a in (approvals_pending.get("items") or [])[:3]:
|
||||
if isinstance(a, dict):
|
||||
activity.append({"type": "approval", "id": a.get("id"), "action": a.get("action_label") or a.get("action"), "status": a.get("status"), "time": a.get("submitted_at")})
|
||||
|
||||
activity.sort(key=lambda x: str(x.get("time") or ""), reverse=True)
|
||||
|
||||
return {
|
||||
"stages": stage_counts,
|
||||
"total_batches": len(batches),
|
||||
"pending_approvals": pending_approvals,
|
||||
"running_jobs": running_jobs,
|
||||
"model_count": model_count,
|
||||
"fleet": summary,
|
||||
"activity": activity[:8],
|
||||
"recent_training": [{"id": r.get("id"), "action": r.get("action"), "status": r.get("status"), "created_at": r.get("created_at")} for r in recent_records if isinstance(r, dict)],
|
||||
}
|
||||
|
||||
|
||||
# ── 世界模型仿真 ──
|
||||
|
||||
class SimulateBody(BaseModel):
|
||||
scene: str = "urban_highway"
|
||||
camera: str = "truck_front"
|
||||
weather: str = "clear"
|
||||
objects: list[str] = Field(default_factory=lambda: ["Pedestrain", "Car", "Truck", "Bus"])
|
||||
density: str = "medium"
|
||||
count: int = 100
|
||||
fov_variant: bool = False
|
||||
note: str = ""
|
||||
|
||||
|
||||
@app.get("/api/v1/simulate/jobs")
|
||||
def api_simulate_jobs(
|
||||
_user: Annotated[User, Depends(get_current_user)],
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
from as_platform.data.simulate import list_jobs
|
||||
return list_jobs(offset=offset, limit=limit)
|
||||
|
||||
|
||||
@app.post("/api/v1/simulate/generate")
|
||||
def api_simulate_generate(
|
||||
body: SimulateBody,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
) -> dict[str, Any]:
|
||||
from as_platform.data.simulate import submit_job
|
||||
return submit_job(body.model_dump(), user_name=user.name)
|
||||
|
||||
|
||||
@app.get("/api/v1/simulate/jobs/{job_id}")
|
||||
def api_simulate_job(
|
||||
job_id: str,
|
||||
_user: Annotated[User, Depends(get_current_user)],
|
||||
) -> dict[str, Any]:
|
||||
from as_platform.data.simulate import get_job
|
||||
job = get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(404, "Job 不存在")
|
||||
return job
|
||||
|
||||
|
||||
@app.get("/api/v1/simulate/jobs/{job_id}/images")
|
||||
def api_simulate_job_images(
|
||||
job_id: str,
|
||||
_user: Annotated[User, Depends(get_current_user)],
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(60, ge=1, le=200),
|
||||
) -> dict[str, Any]:
|
||||
from as_platform.data.simulate import get_job_images
|
||||
return get_job_images(job_id, offset=offset, limit=limit)
|
||||
|
||||
|
||||
@app.post("/api/v1/simulate/jobs/{job_id}/ingest")
|
||||
def api_simulate_ingest(
|
||||
job_id: str,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
task: str = Query("adas"),
|
||||
) -> dict[str, Any]:
|
||||
from as_platform.data.simulate import ingest_job_to_batch
|
||||
return ingest_job_to_batch(job_id, task=task, user_name=user.name)
|
||||
|
||||
|
||||
def _mount_ui() -> None:
|
||||
global _UI_DIR
|
||||
for ui in (PLATFORM_WEB, PLATFORM_DIR / "web" / "dist"):
|
||||
if (ui / "index.html").is_file():
|
||||
_UI_DIR = ui
|
||||
app.mount("/assets", StaticFiles(directory=str(ui / "assets")), name="ui-assets")
|
||||
return
|
||||
ui = PLATFORM_WEB
|
||||
if (ui / "index.html").is_file():
|
||||
_UI_DIR = ui
|
||||
assets = ui / "assets"
|
||||
if assets.is_dir():
|
||||
app.mount("/assets", StaticFiles(directory=str(assets)), name="ui-assets")
|
||||
annotate = ui / "annotate"
|
||||
if annotate.is_dir():
|
||||
app.mount("/annotate", StaticFiles(directory=str(annotate), html=True), name="ui-annotate")
|
||||
return
|
||||
|
||||
|
||||
_mount_ui()
|
||||
|
||||
|
||||
@app.get("/labeling/campaigns/{campaign_id}/annotate", include_in_schema=False)
|
||||
def serve_annotate_app(campaign_id: str):
|
||||
"""标注编辑器页面 — 返回旧 Label Studio 构建产物"""
|
||||
if not _UI_DIR:
|
||||
raise HTTPException(404, "UI not built")
|
||||
annotate_index = _UI_DIR / "annotate" / "index.html"
|
||||
if annotate_index.is_file():
|
||||
return FileResponse(annotate_index)
|
||||
raise HTTPException(404, "标注编辑器未构建")
|
||||
|
||||
|
||||
@app.get("/{full_path:path}", include_in_schema=False)
|
||||
def spa_fallback(full_path: str):
|
||||
if full_path.startswith("api/"):
|
||||
|
||||
381
platform/as_platform/api/system_routes.py
Normal file
381
platform/as_platform/api/system_routes.py
Normal file
@@ -0,0 +1,381 @@
|
||||
"""系统管理 API 路由 — 模块化前缀 /api/v1/system
|
||||
|
||||
本路由将 audit/jobs/traces/agents 功能以模块化路径重新暴露。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from as_platform.agents.tools import TOOL_REGISTRY, invoke_tool
|
||||
from as_platform.agents.trace import get_trace, list_traces
|
||||
from as_platform.audit.queue import (
|
||||
ACTION_LABELS,
|
||||
ACTIONS_REQUIRING_APPROVAL,
|
||||
REJECTION_CATEGORIES,
|
||||
approve_and_execute,
|
||||
batch_approve,
|
||||
batch_reject,
|
||||
get_approval,
|
||||
list_approvals,
|
||||
reject_approval,
|
||||
submit_approval,
|
||||
)
|
||||
from as_platform.audit.preview import find_image_ref, list_scope_images, resolve_approval_scope
|
||||
from as_platform.auth.deps import can_submit_action, get_current_user, require_any_permission, require_permission
|
||||
from as_platform.auth.feishu import sync_feishu_users_to_db
|
||||
from as_platform.auth.users import list_users, list_users_paginated, set_user_roles
|
||||
from as_platform.db.engine import get_db
|
||||
from as_platform.db.init_db import user_to_dict
|
||||
from as_platform.db.models import User
|
||||
from as_platform.jobs.queue import enqueue_job, get_job, list_jobs
|
||||
|
||||
router = APIRouter(prefix="/api/v1/system", tags=["system"])
|
||||
|
||||
|
||||
# ── Audit / Approvals ──
|
||||
|
||||
@router.get("/audit")
|
||||
def api_system_approvals(
|
||||
_user: Annotated[User, Depends(require_permission("read:audit"))],
|
||||
status: str | None = None,
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
return list_approvals(status=status, offset=offset, limit=limit)
|
||||
|
||||
|
||||
@router.get("/audit/actions")
|
||||
def api_system_actions(_user: Annotated[User, Depends(require_permission("read:audit"))]) -> dict[str, Any]:
|
||||
return {"actions": [{"id": k, "label": ACTION_LABELS.get(k, k)} for k in sorted(ACTIONS_REQUIRING_APPROVAL)]}
|
||||
|
||||
|
||||
class SubmitApprovalBody(BaseModel):
|
||||
action: str
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
note: str | None = None
|
||||
|
||||
|
||||
class ReviewBody(BaseModel):
|
||||
comment: str | None = None
|
||||
|
||||
|
||||
class RejectBody(BaseModel):
|
||||
comment: str | None = None
|
||||
rejection_category: str = ""
|
||||
|
||||
|
||||
class BatchReviewBody(BaseModel):
|
||||
ids: list[str] = Field(default_factory=list)
|
||||
comment: str | None = None
|
||||
rejection_category: str = ""
|
||||
|
||||
|
||||
@router.post("/audit/submit")
|
||||
def api_system_submit_approval(
|
||||
body: SubmitApprovalBody,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
) -> dict[str, Any]:
|
||||
if not can_submit_action(user, body.action):
|
||||
raise HTTPException(403, f"无权提交: {body.action}")
|
||||
try:
|
||||
return submit_approval(
|
||||
body.action, body.params,
|
||||
submitted_by=user.name,
|
||||
submitted_by_user_id=user.id,
|
||||
note=body.note,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.get("/audit/{record_id}")
|
||||
def api_system_get_approval(
|
||||
record_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:audit"))],
|
||||
) -> dict[str, Any]:
|
||||
rec = get_approval(record_id)
|
||||
if not rec:
|
||||
raise HTTPException(404, "审核单不存在")
|
||||
return rec
|
||||
|
||||
|
||||
@router.get("/audit/{record_id}/preview")
|
||||
def api_system_approval_preview(
|
||||
record_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:audit"))],
|
||||
) -> dict[str, Any]:
|
||||
rec = get_approval(record_id)
|
||||
if not rec:
|
||||
raise HTTPException(404, "审核单不存在")
|
||||
try:
|
||||
scope = resolve_approval_scope(rec["action"], rec.get("params") or {})
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
return {
|
||||
"approval": rec,
|
||||
"scope_label": scope.get("scope_label"),
|
||||
"task": scope.get("task"),
|
||||
"pack": scope.get("pack"),
|
||||
"class_names": scope.get("class_names"),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/audit/{record_id}/images")
|
||||
def api_system_approval_images(
|
||||
record_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:audit"))],
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(60, ge=1, le=200),
|
||||
) -> dict[str, Any]:
|
||||
rec = get_approval(record_id)
|
||||
if not rec:
|
||||
raise HTTPException(404, "审核单不存在")
|
||||
try:
|
||||
scope = resolve_approval_scope(rec["action"], rec.get("params") or {})
|
||||
return list_scope_images(scope, offset=offset, limit=limit)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.post("/audit/{record_id}/approve")
|
||||
def api_system_approve(
|
||||
record_id: str,
|
||||
body: ReviewBody,
|
||||
user: Annotated[User, Depends(require_permission("write:approval_review"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return approve_and_execute(
|
||||
record_id,
|
||||
reviewed_by=user.name,
|
||||
reviewed_by_user_id=user.id,
|
||||
comment=body.comment,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.get("/audit/rejection-categories")
|
||||
def api_rejection_categories(
|
||||
_user: Annotated[User, Depends(require_permission("read:audit"))],
|
||||
) -> dict[str, Any]:
|
||||
return {"categories": [{"key": k, "label": v} for k, v in REJECTION_CATEGORIES.items()]}
|
||||
|
||||
|
||||
@router.post("/audit/batch-approve")
|
||||
def api_system_batch_approve(
|
||||
body: BatchReviewBody,
|
||||
user: Annotated[User, Depends(require_permission("write:approval_review"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return batch_approve(body.ids, reviewed_by=user.name, reviewed_by_user_id=user.id)
|
||||
except Exception as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.post("/audit/batch-reject")
|
||||
def api_system_batch_reject(
|
||||
body: BatchReviewBody,
|
||||
user: Annotated[User, Depends(require_permission("write:approval_review"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return batch_reject(body.ids, reviewed_by=user.name, reviewed_by_user_id=user.id,
|
||||
comment=body.comment, rejection_category=body.rejection_category)
|
||||
except Exception as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.post("/audit/{record_id}/reject")
|
||||
def api_system_reject(
|
||||
record_id: str,
|
||||
body: RejectBody,
|
||||
user: Annotated[User, Depends(require_permission("write:approval_review"))],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return reject_approval(
|
||||
record_id,
|
||||
reviewed_by=user.name,
|
||||
reviewed_by_user_id=user.id,
|
||||
comment=body.comment,
|
||||
rejection_category=body.rejection_category,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
# ── Jobs ──
|
||||
|
||||
@router.get("/jobs")
|
||||
def api_system_jobs(
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
status: str | None = None,
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
return list_jobs(status=status, offset=offset, limit=limit)
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}")
|
||||
def api_system_job(
|
||||
job_id: str,
|
||||
_user: Annotated[User, Depends(require_permission("read:jobs"))],
|
||||
) -> dict[str, Any]:
|
||||
job = get_job(job_id)
|
||||
if not job:
|
||||
raise HTTPException(404, "Job 不存在")
|
||||
return job
|
||||
|
||||
|
||||
# ── Traces / Execution Logs ──
|
||||
|
||||
@router.get("/traces")
|
||||
def api_system_traces(
|
||||
_user: Annotated[User, Depends(get_current_user)],
|
||||
limit: int = 50,
|
||||
) -> dict[str, Any]:
|
||||
return {"trace_ids": list_traces(limit=limit)}
|
||||
|
||||
|
||||
@router.get("/traces/{trace_id}")
|
||||
def api_system_trace(
|
||||
trace_id: str,
|
||||
_user: Annotated[User, Depends(get_current_user)],
|
||||
) -> dict[str, Any]:
|
||||
spans = get_trace(trace_id)
|
||||
if not spans:
|
||||
raise HTTPException(404, "Trace 不存在")
|
||||
return {"trace_id": trace_id, "spans": spans}
|
||||
|
||||
|
||||
# ── Agent Tools ──
|
||||
|
||||
@router.get("/agents/tools")
|
||||
def api_system_tools(
|
||||
_user: Annotated[User, Depends(get_current_user)],
|
||||
) -> dict[str, Any]:
|
||||
return {"tools": list(TOOL_REGISTRY.keys())}
|
||||
|
||||
|
||||
class ToolInvokeBody(BaseModel):
|
||||
tool: str
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
@router.post("/agents/tools/invoke")
|
||||
def api_system_tool_invoke(
|
||||
body: ToolInvokeBody,
|
||||
_user: Annotated[User, Depends(get_current_user)],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return {"result": invoke_tool(body.tool, **body.params)}
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
# ── User Management ──
|
||||
|
||||
class SetRolesBody(BaseModel):
|
||||
roles: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
@router.get("/users")
|
||||
def api_system_users(
|
||||
_user: Annotated[User, Depends(require_permission("admin:users"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
search: str = Query(""),
|
||||
role: str = Query(""),
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
users, total = list_users_paginated(db, search=search, role_code=role, offset=offset, limit=limit)
|
||||
return {"items": [user_to_dict(u) for u in users], "total": total}
|
||||
|
||||
|
||||
@router.put("/users/{user_id}/roles")
|
||||
def api_system_set_roles(
|
||||
user_id: int,
|
||||
body: SetRolesBody,
|
||||
_user: Annotated[User, Depends(require_permission("admin:users"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict[str, Any]:
|
||||
user = set_user_roles(db, user_id, body.roles)
|
||||
if not user:
|
||||
raise HTTPException(404, "用户不存在")
|
||||
return {"ok": True, "user": user_to_dict(user)}
|
||||
|
||||
|
||||
@router.post("/feishu/sync-users")
|
||||
def api_sync_feishu_users(
|
||||
_user: Annotated[User, Depends(require_permission("admin:users"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
result = sync_feishu_users_to_db(db)
|
||||
db.commit()
|
||||
# 审计日志
|
||||
from as_platform.audit.log_utils import log_op
|
||||
log_op(user_id=_user.id, user_name=_user.name, category="system", action="sync_feishu_users",
|
||||
target_type="user", summary=f"飞书用户同步: 新增{result.get('created',0)} 更新{result.get('updated',0)}")
|
||||
return {"ok": True, **result}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"飞书用户同步失败: {e}") from e
|
||||
|
||||
|
||||
# ── Audit Log ──
|
||||
|
||||
@router.get("/audit-log")
|
||||
def api_audit_log(
|
||||
_user: Annotated[User, Depends(require_permission("admin:users"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
user_id: int | None = None,
|
||||
category: str | None = None,
|
||||
action: str | None = None,
|
||||
search: str = Query(""),
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(20, ge=1, le=200),
|
||||
) -> dict[str, Any]:
|
||||
from as_platform.db.models import OperationLog
|
||||
q = db.query(OperationLog)
|
||||
if user_id:
|
||||
q = q.filter(OperationLog.user_id == user_id)
|
||||
if category:
|
||||
q = q.filter(OperationLog.category == category)
|
||||
if action:
|
||||
q = q.filter(OperationLog.action == action)
|
||||
if search:
|
||||
like = f"%{search}%"
|
||||
from sqlalchemy import or_
|
||||
q = q.filter(or_(OperationLog.summary.ilike(like), OperationLog.user_name.ilike(like)))
|
||||
total = q.count()
|
||||
items = q.order_by(OperationLog.timestamp.desc()).offset(offset).limit(limit).all()
|
||||
return {"items": [log.to_dict() for log in items], "total": total}
|
||||
|
||||
|
||||
@router.get("/audit-log/stats")
|
||||
def api_audit_log_stats(
|
||||
_user: Annotated[User, Depends(require_permission("admin:users"))],
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict[str, Any]:
|
||||
from as_platform.db.models import OperationLog
|
||||
from sqlalchemy import func
|
||||
from datetime import datetime, timezone, timedelta
|
||||
today = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
today_count = db.query(OperationLog).filter(OperationLog.timestamp >= today).count()
|
||||
top_users = (
|
||||
db.query(OperationLog.user_name, func.count().label("cnt"))
|
||||
.filter(OperationLog.timestamp >= today)
|
||||
.group_by(OperationLog.user_name).order_by(func.count().desc()).limit(5).all()
|
||||
)
|
||||
by_category = (
|
||||
db.query(OperationLog.category, func.count().label("cnt"))
|
||||
.filter(OperationLog.timestamp >= today)
|
||||
.group_by(OperationLog.category).all()
|
||||
)
|
||||
return {
|
||||
"today_count": today_count,
|
||||
"top_users": [{"name": name, "count": cnt} for name, cnt in top_users],
|
||||
"by_category": [{"category": cat, "count": cnt} for cat, cnt in by_category],
|
||||
}
|
||||
63
platform/as_platform/audit/log_utils.py
Normal file
63
platform/as_platform/audit/log_utils.py
Normal file
@@ -0,0 +1,63 @@
|
||||
"""操作审计日志工具。异步写入,不阻塞主流程。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from as_platform.db.models import OperationLog
|
||||
|
||||
|
||||
def log_op(
|
||||
*,
|
||||
user_id: int | None = None,
|
||||
user_name: str | None = None,
|
||||
category: str,
|
||||
action: str,
|
||||
target_type: str | None = None,
|
||||
target_id: str | None = None,
|
||||
summary: str = "",
|
||||
detail: dict[str, Any] | None = None,
|
||||
ip_address: str | None = None,
|
||||
) -> None:
|
||||
"""异步记录操作日志。"""
|
||||
threading.Thread(
|
||||
target=_write_log,
|
||||
args=(user_id, user_name, category, action, target_type, target_id, summary, detail, ip_address),
|
||||
daemon=True,
|
||||
name=f"audit-log-{action}",
|
||||
).start()
|
||||
|
||||
|
||||
def _write_log(
|
||||
user_id: int | None,
|
||||
user_name: str | None,
|
||||
category: str,
|
||||
action: str,
|
||||
target_type: str | None,
|
||||
target_id: str | None,
|
||||
summary: str,
|
||||
detail: dict[str, Any] | None,
|
||||
ip_address: str | None,
|
||||
) -> None:
|
||||
try:
|
||||
from as_platform.db.engine import session_scope
|
||||
|
||||
with session_scope() as db:
|
||||
log = OperationLog(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
user_id=user_id,
|
||||
user_name=user_name,
|
||||
category=category,
|
||||
action=action,
|
||||
target_type=target_type,
|
||||
target_id=str(target_id)[:128] if target_id else None,
|
||||
summary=summary[:512] if summary else None,
|
||||
detail_json=json.dumps(detail, ensure_ascii=False) if detail else None,
|
||||
ip_address=ip_address,
|
||||
)
|
||||
db.add(log)
|
||||
db.commit()
|
||||
except Exception:
|
||||
pass # 日志写入失败不影响业务
|
||||
@@ -264,6 +264,33 @@ def resolve_approval_scope(action: str, params: dict[str, Any]) -> dict[str, Any
|
||||
"batches": batches,
|
||||
}
|
||||
|
||||
if action == "delivery_ingest":
|
||||
data_path = (p.get("data_path") or "").strip()
|
||||
if not data_path:
|
||||
raise ValueError("缺少 data_path 参数")
|
||||
src = Path(data_path)
|
||||
project = p.get("project") or "dms"
|
||||
task = p.get("task") or ""
|
||||
batch_name = p.get("batch_name") or src.name
|
||||
scope_label = f"数据送标入湖 · {project}"
|
||||
if task:
|
||||
scope_label += f" · {task}"
|
||||
scope_label += f" · {batch_name}"
|
||||
return {
|
||||
"project": project,
|
||||
"task": task or None,
|
||||
"pack": None,
|
||||
"scope_label": scope_label,
|
||||
"class_names": {},
|
||||
"batches": [
|
||||
{
|
||||
"path": src,
|
||||
"batch": batch_name,
|
||||
"location": "delivery",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
if action in ("train_dms", "promote_dms", "eval_dms"):
|
||||
task = p.get("task")
|
||||
if not task:
|
||||
|
||||
@@ -8,13 +8,27 @@ from typing import Any
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import Approval, User
|
||||
from as_platform.config import LANE_DATA_VIZ_ENABLED
|
||||
from as_platform.integrations.feishu_notify import send_chat_async
|
||||
|
||||
ACTIONS_REQUIRING_APPROVAL = {
|
||||
"build_dms", "build_lane", "enable_pack", "disable_pack",
|
||||
"train_dms", "train_lane", "eval_dms", "promote_dms",
|
||||
"pipeline_dms", "register_batch", "eval_lane", "visualize_dms", "visualize_lane",
|
||||
"delivery_ingest",
|
||||
}
|
||||
|
||||
REJECTION_CATEGORIES = {
|
||||
"data_quality": "数据质量问题",
|
||||
"wrong_params": "参数配置有误",
|
||||
"duplicate": "重复提交",
|
||||
"not_needed": "无需此操作",
|
||||
"permission": "权限不足",
|
||||
"other": "其他原因",
|
||||
}
|
||||
|
||||
REJECTION_CATEGORY_LABEL = {k: v for k, v in REJECTION_CATEGORIES.items()}
|
||||
|
||||
|
||||
ACTION_LABELS = {
|
||||
"build_dms": "DMS 入库 (build)",
|
||||
"build_lane": "车道线合并列表 (build lane)",
|
||||
@@ -29,6 +43,7 @@ ACTION_LABELS = {
|
||||
"promote_dms": "DMS 模型晋级",
|
||||
"pipeline_dms": "DMS 半自动流水线",
|
||||
"register_batch": "登记批次元数据",
|
||||
"delivery_ingest": "数据送标入湖",
|
||||
}
|
||||
|
||||
|
||||
@@ -77,15 +92,36 @@ def submit_approval(
|
||||
|
||||
if auto_execute:
|
||||
return approve_and_execute(out["id"], reviewed_by="system", comment="auto_execute")
|
||||
|
||||
# 飞书通知 + 审计日志
|
||||
label = ACTION_LABELS.get(action, action)
|
||||
send_chat_async(f"📋 新审核提交\n{label}\n提单人: {submitted_by or '—'}\n备注: {note or '—'}")
|
||||
from as_platform.audit.log_utils import log_op
|
||||
log_op(user_id=submitted_by_user_id, user_name=submitted_by or "", category="audit", action="submit_approval",
|
||||
target_type="approval", target_id=out["id"], summary=f"提交审核: {label}",
|
||||
detail={"action": action, "params": params, "note": note})
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def list_approvals(status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
|
||||
def list_approvals(
|
||||
status: str | None = None,
|
||||
*,
|
||||
offset: int = 0,
|
||||
limit: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
q = db.query(Approval).order_by(Approval.submitted_at.desc())
|
||||
if status:
|
||||
q = q.filter(Approval.status == status)
|
||||
return [a.to_dict() for a in q.limit(limit).all()]
|
||||
total = q.count()
|
||||
rows = q.offset(max(0, offset)).limit(max(1, limit)).all()
|
||||
return {
|
||||
"items": [a.to_dict() for a in rows],
|
||||
"total": total,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
|
||||
def get_approval(record_id: str) -> dict[str, Any] | None:
|
||||
@@ -137,6 +173,16 @@ def approve_and_execute(
|
||||
job = enqueue_job(rec["action"], rec.get("params") or {}, approval_id=record_id, async_run=True)
|
||||
|
||||
_update(record_id, job_id=job.get("id"), status="running")
|
||||
|
||||
# 飞书通知 + 审计日志
|
||||
label = ACTION_LABELS.get(rec["action"], rec["action"])
|
||||
submitter = rec.get("submitted_by") or rec.get("submitted_by_name") or ""
|
||||
send_chat_async(f"✅ 审核通过\n{label}\n审核人: {reviewed_by or '—'}\n提单人: {submitter}")
|
||||
from as_platform.audit.log_utils import log_op
|
||||
log_op(user_id=reviewed_by_user_id, user_name=reviewed_by or "", category="audit", action="approve",
|
||||
target_type="approval", target_id=record_id, summary=f"审核通过: {label}",
|
||||
detail={"comment": comment})
|
||||
|
||||
return get_approval(record_id) or {}
|
||||
|
||||
|
||||
@@ -146,17 +192,75 @@ def reject_approval(
|
||||
reviewed_by: str | None = None,
|
||||
reviewed_by_user_id: int | None = None,
|
||||
comment: str | None = None,
|
||||
rejection_category: str = "",
|
||||
) -> dict[str, Any]:
|
||||
rec = get_approval(record_id)
|
||||
if not rec:
|
||||
raise ValueError(f"审核单不存在: {record_id}")
|
||||
if rec.get("status") != "pending":
|
||||
raise ValueError(f"当前状态不可驳回: {rec.get('status')}")
|
||||
return _update(
|
||||
out = _update(
|
||||
record_id,
|
||||
status="rejected",
|
||||
reviewed_by_name=reviewed_by,
|
||||
reviewed_by_user_id=reviewed_by_user_id,
|
||||
reviewed_at=_now(),
|
||||
review_comment=comment,
|
||||
rejection_category=rejection_category or "",
|
||||
) or {}
|
||||
# 飞书通知:审核驳回
|
||||
label = ACTION_LABELS.get(rec["action"], rec["action"])
|
||||
cat = REJECTION_CATEGORIES.get(rejection_category, rejection_category) if rejection_category else ""
|
||||
submitter = rec.get("submitted_by") or rec.get("submitted_by_name") or ""
|
||||
reason_text = f"\n原因: {cat}" if cat else ""
|
||||
send_chat_async(f"❌ 审核驳回\n{label}\n审核人: {reviewed_by or '—'}\n提单人: {submitter}{reason_text}\n意见: {comment or '—'}")
|
||||
from as_platform.audit.log_utils import log_op
|
||||
log_op(user_id=reviewed_by_user_id, user_name=reviewed_by or "", category="audit", action="reject",
|
||||
target_type="approval", target_id=record_id, summary=f"审核驳回: {label}",
|
||||
detail={"comment": comment, "rejection_category": rejection_category})
|
||||
|
||||
try:
|
||||
from as_platform.deliveries.service import mark_delivery_rejected_by_approval
|
||||
mark_delivery_rejected_by_approval(record_id)
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def batch_approve(
|
||||
record_ids: list[str],
|
||||
*,
|
||||
reviewed_by: str | None = None,
|
||||
reviewed_by_user_id: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""批量通过审核。返回 {approved, failed, errors}。"""
|
||||
approved: list[str] = []
|
||||
errors: list[dict[str, str]] = []
|
||||
for rid in record_ids:
|
||||
try:
|
||||
approve_and_execute(rid, reviewed_by=reviewed_by, reviewed_by_user_id=reviewed_by_user_id)
|
||||
approved.append(rid)
|
||||
except Exception as e:
|
||||
errors.append({"id": rid, "error": str(e)})
|
||||
return {"approved": len(approved), "failed": len(errors), "errors": errors}
|
||||
|
||||
|
||||
def batch_reject(
|
||||
record_ids: list[str],
|
||||
*,
|
||||
reviewed_by: str | None = None,
|
||||
reviewed_by_user_id: int | None = None,
|
||||
comment: str | None = None,
|
||||
rejection_category: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""批量驳回审核。"""
|
||||
rejected: list[str] = []
|
||||
errors: list[dict[str, str]] = []
|
||||
for rid in record_ids:
|
||||
try:
|
||||
reject_approval(rid, reviewed_by=reviewed_by, reviewed_by_user_id=reviewed_by_user_id,
|
||||
comment=comment, rejection_category=rejection_category)
|
||||
rejected.append(rid)
|
||||
except Exception as e:
|
||||
errors.append({"id": rid, "error": str(e)})
|
||||
return {"rejected": len(rejected), "failed": len(errors), "errors": errors}
|
||||
|
||||
286
platform/as_platform/audit/review.py
Normal file
286
platform/as_platform/audit/review.py
Normal file
@@ -0,0 +1,286 @@
|
||||
"""标注质检 — 逐张审核标注质量(Good/Fine/Bad 评分 + PIL 优化渲染)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
|
||||
|
||||
from as_platform.data.batch import IMG_EXTS
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import Base
|
||||
|
||||
IMAGE_EXTS = tuple(ext.lower() for ext in IMG_EXTS)
|
||||
|
||||
# ── PIL font cache ──
|
||||
_font_cache: dict[int, ImageFont.FreeTypeFont | ImageFont.ImageFont] = {}
|
||||
|
||||
def _get_font(size: int) -> ImageFont.FreeTypeFont | ImageFont.ImageFont:
|
||||
if size not in _font_cache:
|
||||
try:
|
||||
_font_cache[size] = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", size)
|
||||
except Exception:
|
||||
try:
|
||||
_font_cache[size] = ImageFont.truetype("/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", size)
|
||||
except Exception:
|
||||
_font_cache[size] = ImageFont.load_default()
|
||||
return _font_cache[size]
|
||||
|
||||
|
||||
# ── YOLO bbox utils ──
|
||||
|
||||
def _parse_yolo_line(line: str) -> dict[str, Any] | None:
|
||||
parts = line.strip().split()
|
||||
if len(parts) < 5:
|
||||
return None
|
||||
try:
|
||||
return {"class_id": int(float(parts[0])), "bbox": tuple(map(float, parts[1:5]))}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _bbox_to_xyxy(bbox: tuple[float, ...], w: int, h: int) -> tuple[int, int, int, int]:
|
||||
cx, cy, bw, bh = bbox[:4]
|
||||
x1 = int((cx - bw / 2) * w)
|
||||
y1 = int((cy - bh / 2) * h)
|
||||
x2 = int((cx + bw / 2) * w)
|
||||
y2 = int((cy + bh / 2) * h)
|
||||
return max(0, x1), max(0, y1), min(w, x2), min(h, y2)
|
||||
|
||||
|
||||
def _parse_labels(label_path: Path) -> list[dict[str, Any]]:
|
||||
if not label_path or not label_path.is_file():
|
||||
return []
|
||||
results = []
|
||||
for line in label_path.read_text().strip().splitlines():
|
||||
ann = _parse_yolo_line(line)
|
||||
if ann:
|
||||
results.append(ann)
|
||||
return results
|
||||
|
||||
|
||||
# ── Optimized overlay render ──
|
||||
|
||||
PALETTE = [(220, 20, 60), (30, 144, 255), (50, 205, 50), (255, 165, 0), (186, 85, 211), (0, 206, 209)]
|
||||
|
||||
|
||||
def render_review_overlay(
|
||||
image_path: Path,
|
||||
label_path: Path | None,
|
||||
class_names: dict[int, str],
|
||||
*,
|
||||
max_size: int = 800,
|
||||
quality: int = 85,
|
||||
) -> bytes:
|
||||
"""PIL optimized: single pass resize + draw, no copy. Returns JPEG bytes."""
|
||||
with Image.open(image_path) as im:
|
||||
if im.mode != "RGB":
|
||||
im = im.convert("RGB")
|
||||
# Resize first for faster drawing
|
||||
if max_size and max(im.size) > max_size:
|
||||
im.thumbnail((max_size, max_size), Image.Resampling.LANCZOS)
|
||||
w, h = im.size
|
||||
draw = ImageDraw.Draw(im)
|
||||
font = _get_font(max(12, min(16, w // 50)))
|
||||
line_w = max(1, w // 400)
|
||||
|
||||
anns = _parse_labels(label_path) if label_path else []
|
||||
for ann in anns:
|
||||
cid = ann["class_id"]
|
||||
color = PALETTE[cid % len(PALETTE)]
|
||||
x1, y1, x2, y2 = _bbox_to_xyxy(ann["bbox"], w, h)
|
||||
draw.rectangle((x1, y1, x2, y2), outline=color, width=line_w)
|
||||
label = class_names.get(cid, f"cls_{cid}")
|
||||
draw.text((x1 + 2, max(0, y1 - 16)), label, fill=color, font=font)
|
||||
|
||||
buf = io.BytesIO()
|
||||
im.save(buf, format="JPEG", quality=quality)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
# ── Quality Review Model ──
|
||||
|
||||
class LabelingReview(Base):
|
||||
__tablename__ = "labeling_reviews"
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
campaign_id = Column(String(64), nullable=False, index=True)
|
||||
image_path = Column(String(512), nullable=False)
|
||||
score = Column(String(16), nullable=False, default="pending") # good / fine / bad
|
||||
reviewer_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
|
||||
reviewer_name = Column(String(128), nullable=True)
|
||||
comment = Column(Text, nullable=True)
|
||||
reviewed_at = Column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"campaign_id": self.campaign_id,
|
||||
"image_path": self.image_path,
|
||||
"score": self.score,
|
||||
"reviewer_user_id": self.reviewer_user_id,
|
||||
"reviewer_name": self.reviewer_name,
|
||||
"comment": self.comment,
|
||||
"reviewed_at": self.reviewed_at.isoformat() if self.reviewed_at else None,
|
||||
}
|
||||
|
||||
|
||||
# ── Review operations ──
|
||||
|
||||
def get_review_queue(campaign_id: str, offset: int = 0, limit: int = 20) -> dict[str, Any]:
|
||||
from as_platform.labeling.annotate import resolve_campaign_batch_dir
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign
|
||||
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
return {"items": [], "total": 0, "hint": "Campaign 不存在"}
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
if not batch_dir or not batch_dir.is_dir():
|
||||
return {"items": [], "total": 0, "hint": "批次目录不存在"}
|
||||
|
||||
img_dir = batch_dir / "images"
|
||||
if not img_dir.is_dir():
|
||||
return {"items": [], "total": 0, "hint": "无 images 目录"}
|
||||
|
||||
all_images: list[Path] = []
|
||||
for ext in IMAGE_EXTS:
|
||||
all_images.extend(sorted(img_dir.rglob(f"*{ext}")))
|
||||
|
||||
# Get existing reviews
|
||||
with session_scope() as db:
|
||||
reviewed = {
|
||||
r.image_path: r.score
|
||||
for r in db.query(LabelingReview).filter(LabelingReview.campaign_id == campaign_id).all()
|
||||
}
|
||||
|
||||
total = len(all_images)
|
||||
page = all_images[offset:offset + limit]
|
||||
score_counts = {"good": 0, "fine": 0, "bad": 0, "pending": 0}
|
||||
items = []
|
||||
for img in page:
|
||||
rel = str(img.relative_to(batch_dir))
|
||||
score = reviewed.get(rel, "pending")
|
||||
score_counts[score] += 1
|
||||
label_path = batch_dir / "labels" / (img.stem + ".txt")
|
||||
items.append({
|
||||
"id": rel, "image_path": rel,
|
||||
"fileName": img.name,
|
||||
"score": score,
|
||||
"has_label": label_path.is_file(),
|
||||
})
|
||||
|
||||
# Fill remaining counts
|
||||
for img in all_images:
|
||||
rel = str(img.relative_to(batch_dir))
|
||||
s = reviewed.get(rel, "pending")
|
||||
if s not in score_counts:
|
||||
score_counts[s] = 0
|
||||
|
||||
return {
|
||||
"items": items, "total": total,
|
||||
"offset": offset, "limit": limit,
|
||||
"scores": score_counts,
|
||||
}
|
||||
|
||||
|
||||
def get_review_image(campaign_id: str, image_rel_path: str, class_names: dict[int, str]) -> bytes:
|
||||
from as_platform.labeling.annotate import resolve_campaign_batch_dir
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("Campaign 不存在")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
if not batch_dir:
|
||||
raise FileNotFoundError("批次不存在")
|
||||
img_path = batch_dir / image_rel_path
|
||||
lbl_path = batch_dir / "labels" / (img_path.stem + ".txt")
|
||||
return render_review_overlay(img_path, lbl_path if lbl_path.is_file() else None, class_names)
|
||||
|
||||
|
||||
def submit_review_scores(
|
||||
campaign_id: str,
|
||||
scores: list[dict[str, str]],
|
||||
reviewer_user_id: int | None = None,
|
||||
reviewer_name: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = datetime.now(timezone.utc)
|
||||
updated = 0
|
||||
with session_scope() as db:
|
||||
for item in scores:
|
||||
img_path = item["image_path"]
|
||||
score = item["score"]
|
||||
rec = db.query(LabelingReview).filter(
|
||||
LabelingReview.campaign_id == campaign_id,
|
||||
LabelingReview.image_path == img_path,
|
||||
).first()
|
||||
if rec:
|
||||
rec.score = score
|
||||
rec.reviewer_user_id = reviewer_user_id
|
||||
rec.reviewer_name = reviewer_name
|
||||
rec.reviewed_at = now
|
||||
rec.comment = item.get("comment")
|
||||
else:
|
||||
db.add(LabelingReview(
|
||||
campaign_id=campaign_id, image_path=img_path, score=score,
|
||||
reviewer_user_id=reviewer_user_id, reviewer_name=reviewer_name,
|
||||
reviewed_at=now, comment=item.get("comment"),
|
||||
))
|
||||
updated += 1
|
||||
db.commit()
|
||||
|
||||
# Check if all images are reviewed and auto-advance stage
|
||||
counts = _review_db_counts(db, campaign_id)
|
||||
from as_platform.labeling.annotate import resolve_campaign_batch_dir
|
||||
from as_platform.data.batch import IMG_EXTS
|
||||
from as_platform.db.engine import session_scope as _scope
|
||||
from as_platform.db.models import LabelingCampaign as _LC
|
||||
with _scope() as _db:
|
||||
_camp = _db.get(_LC, campaign_id)
|
||||
batch_dir = resolve_campaign_batch_dir(_camp) if _camp else None
|
||||
total_images = 0
|
||||
if batch_dir and (batch_dir / "images").is_dir():
|
||||
for ext in IMG_EXTS:
|
||||
total_images += len(list((batch_dir / "images").rglob(f"*{ext}")))
|
||||
|
||||
reviewed = sum(counts.values())
|
||||
if reviewed >= total_images and total_images > 0:
|
||||
pass_rate = counts.get("good", 0) / max(total_images, 1)
|
||||
new_stage = "review_approved" if pass_rate >= 0.8 else "review_rejected"
|
||||
_update_campaign_stage(db, campaign_id, new_stage)
|
||||
|
||||
return {"ok": True, "updated": updated, "auto_advanced": reviewed >= total_images if total_images > 0 else False}
|
||||
|
||||
|
||||
def _review_db_counts(db, campaign_id: str) -> dict[str, int]:
|
||||
from sqlalchemy import func
|
||||
from collections import Counter
|
||||
rows = db.query(LabelingReview.score, func.count()).filter(
|
||||
LabelingReview.campaign_id == campaign_id
|
||||
).group_by(LabelingReview.score).all()
|
||||
return {score: cnt for score, cnt in rows}
|
||||
|
||||
|
||||
def _update_campaign_stage(db, campaign_id: str, new_stage: str) -> None:
|
||||
from as_platform.db.models import LabelingCampaign
|
||||
from as_platform.labeling.batch_stage import update_campaign_batch_meta_stage
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if camp:
|
||||
camp.status = new_stage
|
||||
db.flush()
|
||||
update_campaign_batch_meta_stage(camp, new_stage)
|
||||
|
||||
|
||||
def review_progress(campaign_id: str) -> dict[str, int]:
|
||||
with session_scope() as db:
|
||||
rows = db.query(LabelingReview).filter(LabelingReview.campaign_id == campaign_id).all()
|
||||
counts = {"good": 0, "fine": 0, "bad": 0, "pending": 0}
|
||||
for r in rows:
|
||||
counts[r.score] = counts.get(r.score, 0) + 1
|
||||
return counts
|
||||
@@ -68,4 +68,6 @@ def can_submit_action(user: User, action: str) -> bool:
|
||||
return True
|
||||
if action == "register_batch" and user_has_permission(user, "write:approval_submit:register"):
|
||||
return True
|
||||
if action == "delivery_ingest" and user_has_permission(user, "write:delivery_submit"):
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -16,6 +16,8 @@ FEISHU_TOKEN_URL = "https://open.feishu.cn/open-apis/authen/v1/access_token"
|
||||
FEISHU_USER_URL = "https://open.feishu.cn/open-apis/authen/v1/user_info"
|
||||
FEISHU_TENANT_TOKEN_URL = "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal"
|
||||
FEISHU_CONTACT_USER_URL = "https://open.feishu.cn/open-apis/contact/v3/users/{user_id}"
|
||||
FEISHU_CONTACT_USERS_URL = "https://open.feishu.cn/open-apis/contact/v3/users"
|
||||
FEISHU_DEPARTMENTS_URL = "https://open.feishu.cn/open-apis/contact/v3/departments"
|
||||
|
||||
STATE_ALG = "HS256"
|
||||
STATE_EXPIRE_MINUTES = 10
|
||||
@@ -140,3 +142,102 @@ def _get_contact_user_profile(
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "飞书联系人信息获取失败")
|
||||
return data.get("data", {}).get("user", {})
|
||||
|
||||
|
||||
def fetch_feishu_users(page_size: int = 100) -> tuple[list[dict[str, Any]], dict[str, str]]:
|
||||
"""从飞书通讯录批量拉取用户列表并获取部门名称映射,返回 (users, dept_name_map)。
|
||||
如果应用无通讯录权限则返回空列表。"""
|
||||
if not is_feishu_configured():
|
||||
return [], {}
|
||||
|
||||
try:
|
||||
with httpx.Client(timeout=30) as client:
|
||||
token = _get_tenant_access_token(client)
|
||||
|
||||
# 先获取部门列表(如果应用有通讯录权限)
|
||||
dept_map: dict[str, str] = {}
|
||||
try:
|
||||
page_token = ""
|
||||
while True:
|
||||
params: dict[str, str | int] = {"page_size": 100, "department_id_type": "open_department_id"}
|
||||
if page_token:
|
||||
params["page_token"] = page_token
|
||||
resp = client.get(FEISHU_DEPARTMENTS_URL, params=params, headers={"Authorization": f"Bearer {token}"})
|
||||
if resp.status_code >= 400:
|
||||
break # 无部门权限,跳过
|
||||
d = resp.json()
|
||||
if d.get("code") == 0:
|
||||
for item in d.get("data", {}).get("items", []):
|
||||
dept_map[item.get("open_department_id", "")] = item.get("name", "")
|
||||
page_token = d.get("data", {}).get("page_token", "")
|
||||
if not page_token:
|
||||
break
|
||||
except Exception:
|
||||
pass # 无需部门信息也可同步用户
|
||||
|
||||
# 再获取用户列表(如无通讯录权限则跳过)
|
||||
users: list[dict[str, Any]] = []
|
||||
try:
|
||||
page_token = ""
|
||||
while True:
|
||||
params = {
|
||||
"user_id_type": "open_id",
|
||||
"department_id_type": "open_department_id",
|
||||
"page_size": min(page_size, 100),
|
||||
}
|
||||
if page_token:
|
||||
params["page_token"] = page_token # type: ignore
|
||||
resp = client.get(FEISHU_CONTACT_USERS_URL, params=params, headers={"Authorization": f"Bearer {token}"}) # type: ignore
|
||||
if resp.status_code >= 400:
|
||||
break
|
||||
d = resp.json()
|
||||
if d.get("code") == 0:
|
||||
for item in d.get("data", {}).get("items", []):
|
||||
users.append({
|
||||
"open_id": item.get("open_id"),
|
||||
"union_id": item.get("union_id"),
|
||||
"user_id": item.get("user_id"),
|
||||
"name": item.get("name"),
|
||||
"email": item.get("email"),
|
||||
"mobile": item.get("mobile"),
|
||||
"avatar_url": item.get("avatar", {}).get("avatar_240") if isinstance(item.get("avatar"), dict) else None,
|
||||
"department_ids": item.get("department_ids", []),
|
||||
})
|
||||
page_token = d.get("data", {}).get("page_token", "")
|
||||
if not page_token:
|
||||
break
|
||||
except Exception:
|
||||
pass # 无通讯录权限,跳过用户列表拉取
|
||||
|
||||
return users, dept_map
|
||||
except Exception:
|
||||
return [], {}
|
||||
|
||||
|
||||
def sync_feishu_users_to_db(db: Session) -> dict[str, int]:
|
||||
"""同步飞书用户到数据库,返回 {created, updated, total}。"""
|
||||
users, dept_map = fetch_feishu_users()
|
||||
created, updated = 0, 0
|
||||
for info in users:
|
||||
open_id = info.get("open_id")
|
||||
if not open_id:
|
||||
continue
|
||||
user = db.query(User).filter(User.feishu_open_id == open_id).first()
|
||||
if not user:
|
||||
user = User(feishu_open_id=open_id)
|
||||
db.add(user)
|
||||
created += 1
|
||||
else:
|
||||
updated += 1
|
||||
user.feishu_union_id = info.get("union_id") or user.feishu_union_id
|
||||
user.feishu_user_id = info.get("user_id") or user.feishu_user_id
|
||||
user.name = info.get("name") or user.name
|
||||
user.email = info.get("email") or user.email or user.email
|
||||
user.avatar_url = info.get("avatar_url") or user.avatar_url
|
||||
dept_ids = info.get("department_ids")
|
||||
if isinstance(dept_ids, list) and dept_ids:
|
||||
dept_names = [dept_map.get(str(did), str(did)) for did in dept_ids]
|
||||
user.feishu_department_ids_json = json.dumps(dept_names, ensure_ascii=False)
|
||||
user.is_active = True
|
||||
db.flush()
|
||||
return {"created": created, "updated": updated, "total": len(users)}
|
||||
|
||||
@@ -58,6 +58,24 @@ def list_users(db: Session) -> list[User]:
|
||||
return db.query(User).options(joinedload(User.roles)).order_by(User.id).all()
|
||||
|
||||
|
||||
def list_users_paginated(
|
||||
db: Session,
|
||||
search: str = "",
|
||||
role_code: str = "",
|
||||
offset: int = 0,
|
||||
limit: int = 20,
|
||||
) -> tuple[list[User], int]:
|
||||
q = db.query(User).options(joinedload(User.roles))
|
||||
if search:
|
||||
like = f"%{search}%"
|
||||
q = q.filter((User.name.ilike(like)) | (User.email.ilike(like)))
|
||||
if role_code:
|
||||
q = q.join(User.roles).filter(Role.code == role_code)
|
||||
total = q.count()
|
||||
users = q.order_by(User.id).offset(offset).limit(limit).all()
|
||||
return users, total
|
||||
|
||||
|
||||
def set_user_roles(db: Session, user_id: int, role_codes: list[str]) -> User | None:
|
||||
user = get_user_by_id(db, user_id)
|
||||
if not user:
|
||||
|
||||
@@ -8,8 +8,29 @@ from urllib.parse import quote_plus
|
||||
AS_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
WORKSPACE = AS_ROOT
|
||||
|
||||
|
||||
def resolve_workspace_root() -> Path | None:
|
||||
"""外部 workspace(DMS/Lane 大文件);Docker 内通常为 /data/workspace。"""
|
||||
candidates: list[Path] = []
|
||||
explicit = os.environ.get("AS_WORKSPACE_ROOT", "").strip()
|
||||
if explicit:
|
||||
candidates.append(Path(explicit))
|
||||
candidates.extend((Path("/data/workspace"), AS_ROOT.parent / "workspace"))
|
||||
for cand in candidates:
|
||||
try:
|
||||
resolved = cand.resolve()
|
||||
except OSError:
|
||||
continue
|
||||
if resolved.is_dir() and (resolved / "DMS").is_dir():
|
||||
return resolved
|
||||
return None
|
||||
|
||||
|
||||
WORKSPACE_ROOT = resolve_workspace_root()
|
||||
|
||||
PLATFORM_DIR = AS_ROOT / "platform"
|
||||
PLATFORM_WEB = PLATFORM_DIR / "web" / "dist"
|
||||
# Label Studio 工程构建产物(scripts/build_hsap_ls_ui.sh)
|
||||
PLATFORM_WEB = PLATFORM_DIR / "ui-hsap" / "dist"
|
||||
ALGORITHMS_DIR = AS_ROOT / "algorithms"
|
||||
DATASETS_DIR = AS_ROOT / "datasets"
|
||||
ALGORITHMS_REGISTRY = ALGORITHMS_DIR / "registry.yaml"
|
||||
@@ -59,6 +80,7 @@ FEISHU_REDIRECT_URI = os.environ.get(
|
||||
)
|
||||
FRONTEND_URL = os.environ.get("AS_FRONTEND_URL", "http://127.0.0.1:8787")
|
||||
DEV_AUTH_ENABLED = os.environ.get("AS_DEV_AUTH", "").lower() in ("1", "true", "yes")
|
||||
FORCE_DEV_AUTH = os.environ.get("AS_FORCE_DEV_AUTH", "").lower() in ("1", "true", "yes")
|
||||
FEISHU_ADMIN_OPEN_IDS = {
|
||||
x.strip() for x in os.environ.get("FEISHU_ADMIN_OPEN_IDS", "").split(",") if x.strip()
|
||||
}
|
||||
@@ -66,6 +88,71 @@ FEISHU_ADMIN_DEPARTMENT_IDS = {
|
||||
x.strip() for x in os.environ.get("FEISHU_ADMIN_DEPARTMENT_IDS", "").split(",") if x.strip()
|
||||
}
|
||||
|
||||
# ── 飞书多维表格(内网出站同步)──
|
||||
FEISHU_BITABLE_APP_TOKEN = os.environ.get("FEISHU_BITABLE_APP_TOKEN", "").strip()
|
||||
# 表在知识库/wiki 内时:填 wiki 链接里 /wiki/ 后节点 ID;HSAP 会调 API 解析为 Basc obj_token
|
||||
FEISHU_BITABLE_WIKI_NODE_TOKEN = os.environ.get("FEISHU_BITABLE_WIKI_NODE_TOKEN", "").strip()
|
||||
FEISHU_BITABLE_TABLE_ID = os.environ.get("FEISHU_BITABLE_TABLE_ID", "").strip()
|
||||
FEISHU_LABELING_CHAT_ID = os.environ.get("FEISHU_LABELING_CHAT_ID", "").strip()
|
||||
FEISHU_BITABLE_SYNC_ENABLED = os.environ.get("FEISHU_BITABLE_SYNC_ENABLED", "").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
FEISHU_BITABLE_SYNC_INTERVAL_SEC = max(30, int(os.environ.get("FEISHU_BITABLE_SYNC_INTERVAL_SEC", "120")))
|
||||
FEISHU_BITABLE_AUTO_INGEST = os.environ.get("FEISHU_BITABLE_AUTO_INGEST", "").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
FEISHU_BITABLE_WEBHOOK_ENABLED = os.environ.get("FEISHU_BITABLE_WEBHOOK_ENABLED", "").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
|
||||
# 中文列名(与 FEISHU_BITABLE_OPS.md 一致,可用环境变量覆盖)
|
||||
def _field(name: str, default: str) -> str:
|
||||
return os.environ.get(name, default).strip() or default
|
||||
|
||||
|
||||
FEISHU_BITABLE_FIELDS: dict[str, str] = {
|
||||
"delivery_id": _field("FEISHU_BITABLE_FIELD_DELIVERY_ID", "批次编号"),
|
||||
"project": _field("FEISHU_BITABLE_FIELD_PROJECT", "项目"),
|
||||
"task": _field("FEISHU_BITABLE_FIELD_TASK", "任务"),
|
||||
"mode": _field("FEISHU_BITABLE_FIELD_MODE", "子模式"),
|
||||
"batch_name": _field("FEISHU_BITABLE_FIELD_BATCH_NAME", "批次名"),
|
||||
"data_path": _field("FEISHU_BITABLE_FIELD_DATA_PATH", "数据路径"),
|
||||
"status": _field("FEISHU_BITABLE_FIELD_STATUS", "状态"),
|
||||
"candidate_id": _field("FEISHU_BITABLE_FIELD_CANDIDATE_ID", "候选ID"),
|
||||
"campaign_id": _field("FEISHU_BITABLE_FIELD_CAMPAIGN_ID", "活动ID"),
|
||||
"inbox_path": _field("FEISHU_BITABLE_FIELD_INBOX_PATH", "Inbox路径"),
|
||||
"progress": _field("FEISHU_BITABLE_FIELD_PROGRESS", "HSAP进度"),
|
||||
"hsap_link": _field("FEISHU_BITABLE_FIELD_HSAP_LINK", "HSAP链接"),
|
||||
"error_message": _field("FEISHU_BITABLE_FIELD_ERROR_MESSAGE", "失败原因"),
|
||||
"last_sync": _field("FEISHU_BITABLE_FIELD_LAST_SYNC", "最后同步"),
|
||||
"record_id": _field("FEISHU_BITABLE_FIELD_RECORD_ID", "记录ID"),
|
||||
}
|
||||
|
||||
# HSAP stage → 飞书表状态
|
||||
FEISHU_STATUS_FROM_STAGE: dict[str, str] = {
|
||||
"raw_pool": "待送标",
|
||||
"out_for_labeling": "标注中",
|
||||
"returned": "待入库",
|
||||
"ingested": "已入库",
|
||||
"labeling_submitted": "标注中",
|
||||
}
|
||||
|
||||
# ── 功能开关 ──
|
||||
# 车道线 catalog 质检统计(mask 抽样,较慢);暂时默认关闭
|
||||
LANE_DATA_VIZ_ENABLED = os.environ.get("AS_LANE_DATA_VIZ", "").lower() in ("1", "true", "yes")
|
||||
|
||||
# ── 车队地图 / T-Box ──
|
||||
FLEET_MAP_ENABLED = os.environ.get("AS_FLEET_MAP_ENABLED", "1").lower() in ("1", "true", "yes")
|
||||
FLEET_MOCK_SEED = os.environ.get("AS_FLEET_MOCK_SEED", "1").lower() in ("1", "true", "yes")
|
||||
FLEET_MOCK_SIMULATE = os.environ.get("AS_FLEET_MOCK_SIMULATE", "1").lower() in ("1", "true", "yes")
|
||||
FLEET_SIM_INTERVAL_SEC = int(os.environ.get("AS_FLEET_SIM_INTERVAL_SEC", "8"))
|
||||
AMAP_KEY = os.environ.get("AS_AMAP_KEY", "").strip()
|
||||
# 车队地图瓦片:gaode(国内默认)| osm
|
||||
MAP_TILE_PROVIDER = os.environ.get("AS_MAP_TILE_PROVIDER", "gaode").strip().lower() or "gaode"
|
||||
TBOX_INGEST_TOKEN = os.environ.get("AS_TBOX_INGEST_TOKEN", "hsap-demo-tbox-token").strip()
|
||||
|
||||
@@ -15,6 +15,7 @@ from as_platform.data.lake import (
|
||||
get_candidate,
|
||||
link_candidate_analysis_job,
|
||||
list_candidates,
|
||||
promote_candidate_to_inbox,
|
||||
write_candidate_upload,
|
||||
)
|
||||
|
||||
@@ -34,4 +35,5 @@ __all__ = [
|
||||
"get_candidate",
|
||||
"link_candidate_analysis_job",
|
||||
"analyze_uploaded_candidate",
|
||||
"promote_candidate_to_inbox",
|
||||
]
|
||||
|
||||
@@ -13,7 +13,7 @@ from as_platform.config import WORKSPACE
|
||||
CATALOG_CACHE_FILE = WORKSPACE / "manifests" / "catalog_cache.json"
|
||||
CATALOG_CACHE_TTL_SEC = int(os.environ.get("AS_CATALOG_CACHE_TTL_SEC", "300"))
|
||||
CATALOG_USE_REPORTS = os.environ.get("AS_CATALOG_USE_REPORTS", "1").lower() in ("1", "true", "yes")
|
||||
CATALOG_CACHE_VERSION = 3
|
||||
CATALOG_CACHE_VERSION = 7
|
||||
|
||||
REPORTS_DIR = WORKSPACE / "reports"
|
||||
DMS_SUMMARY_CSV = REPORTS_DIR / "dms_task_image_summary.csv"
|
||||
@@ -111,8 +111,14 @@ def build_catalog_signature(wf: dict, proj_root_fn) -> dict[str, Any]:
|
||||
import yaml
|
||||
|
||||
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
|
||||
for task in (reg.get("tasks") or {}).keys():
|
||||
dirs.append(_dir_fingerprint(root / "inbox" / task))
|
||||
for task, tcfg in (reg.get("tasks") or {}).items():
|
||||
if tcfg.get("type") == "multi":
|
||||
for mcfg in (tcfg.get("modes") or {}).values():
|
||||
rel = mcfg.get("inbox", "")
|
||||
if rel:
|
||||
dirs.append(_dir_fingerprint(root / rel))
|
||||
else:
|
||||
dirs.append(_dir_fingerprint(root / (tcfg.get("inbox") or f"inbox/{task}")))
|
||||
if pname == "lane":
|
||||
dirs.append(_dir_fingerprint(_pack_registry_path("lane", root, wf), scan_children=False))
|
||||
|
||||
@@ -140,13 +146,41 @@ def save_disk_cache(payload: dict[str, Any]) -> None:
|
||||
CATALOG_CACHE_FILE.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
|
||||
def _iter_dms_packs(task_entry: dict[str, Any]):
|
||||
modes = task_entry.get("modes")
|
||||
if isinstance(modes, dict):
|
||||
for mode_entry in modes.values():
|
||||
for pack in mode_entry.get("packs") or []:
|
||||
if isinstance(pack, dict):
|
||||
yield pack, mode_entry.get("type")
|
||||
return
|
||||
for pack in task_entry.get("packs") or []:
|
||||
if isinstance(pack, dict):
|
||||
yield pack, task_entry.get("type")
|
||||
|
||||
|
||||
def _entry_needs_bbox_refresh(entry: dict[str, Any]) -> bool:
|
||||
if entry.get("type") == "classify":
|
||||
return False
|
||||
has_class = bool(entry.get("class_counts"))
|
||||
any_pts = False
|
||||
any_boxes = False
|
||||
for pack in entry.get("packs") or []:
|
||||
if pack.get("bbox_points"):
|
||||
any_pts = True
|
||||
if int(pack.get("total_boxes") or 0) > 0:
|
||||
any_boxes = True
|
||||
return not any_pts and (any_boxes or has_class)
|
||||
|
||||
|
||||
def _catalog_has_empty_bbox(catalog: dict[str, Any]) -> bool:
|
||||
for task in (catalog.get("dms") or {}).values():
|
||||
for pack in task.get("packs") or []:
|
||||
boxes = int(pack.get("total_boxes") or 0)
|
||||
pts = pack.get("bbox_points") or []
|
||||
if boxes > 0 and not pts:
|
||||
return True
|
||||
if task.get("type") == "multi":
|
||||
for mode_entry in (task.get("modes") or {}).values():
|
||||
if _entry_needs_bbox_refresh(mode_entry):
|
||||
return True
|
||||
elif _entry_needs_bbox_refresh(task):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -198,7 +232,8 @@ def store_catalog_cache(signature: dict[str, Any], data: dict[str, Any], *, buil
|
||||
"data": data,
|
||||
}
|
||||
_CATALOG_MEM_CACHE = payload
|
||||
save_disk_cache(payload)
|
||||
if not _catalog_has_empty_bbox(data):
|
||||
save_disk_cache(payload)
|
||||
return payload
|
||||
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from as_platform.config import WORKSPACE, LANE_DATA_VIZ_ENABLED
|
||||
from as_platform.data.batch import META_FILENAME, enrich_batch, write_meta
|
||||
from as_platform.config import WORKSPACE, WORKSPACE_ROOT, LANE_DATA_VIZ_ENABLED
|
||||
from as_platform.data.batch import META_FILENAME, dms_has_images, enrich_batch, write_meta
|
||||
from as_platform.data.catalog_cache import (
|
||||
build_catalog_signature,
|
||||
get_cached_catalog,
|
||||
@@ -20,6 +20,30 @@ from as_platform.data.catalog_cache import (
|
||||
store_catalog_cache,
|
||||
)
|
||||
|
||||
import sys as _sys
|
||||
|
||||
def _dms_scripts_dir() -> Path:
|
||||
for cand in (
|
||||
WORKSPACE / "datasets" / "dms" / "scripts",
|
||||
WORKSPACE / "datasets" / "dms.embedded.bak" / "scripts",
|
||||
):
|
||||
if (cand / "task_registry.py").is_file():
|
||||
return cand
|
||||
return WORKSPACE / "datasets" / "dms" / "scripts"
|
||||
|
||||
|
||||
_DMS_SCRIPTS = _dms_scripts_dir()
|
||||
if str(_DMS_SCRIPTS) not in _sys.path:
|
||||
_sys.path.insert(0, str(_DMS_SCRIPTS))
|
||||
from task_registry import ( # noqa: E402
|
||||
DOMAIN_LABELS,
|
||||
get_mode_config,
|
||||
inbox_dir,
|
||||
iter_catalog_tasks,
|
||||
report_task_key,
|
||||
task_defs_for_pending,
|
||||
)
|
||||
|
||||
MAX_LABEL_FILES_PER_PACK = 2000
|
||||
MAX_BBOX_POINTS_PER_PACK = 1500
|
||||
MAX_LANE_MASK_SAMPLES_PER_PACK = 500
|
||||
@@ -60,8 +84,36 @@ def resolve_pack(project: str, root: Path, wf: dict, name: str) -> str:
|
||||
raise ValueError(f"[{project}] 未知包: {name},已登记: {known}")
|
||||
|
||||
|
||||
def _pack_dir_usable(path: Path) -> bool:
|
||||
try:
|
||||
return path.is_dir() and any(path.iterdir())
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _dms_workspace_pack_dir(pack_name: str) -> Path | None:
|
||||
ws = WORKSPACE_ROOT
|
||||
if ws is None:
|
||||
return None
|
||||
cand = ws / "DMS/DATASET/packs" / pack_name
|
||||
return cand if _pack_dir_usable(cand) else None
|
||||
|
||||
|
||||
def resolve_pack_dir(project: str, root: Path, wf: dict, name: str) -> Path:
|
||||
return (root / resolve_pack(project, root, wf, name)).resolve()
|
||||
rel = resolve_pack(project, root, wf, name)
|
||||
primary = root / rel
|
||||
if _pack_dir_usable(primary):
|
||||
return primary.resolve()
|
||||
reg = load_pack_registry(project, root, wf)
|
||||
pack_name = reg.get("aliases", {}).get(name, name)
|
||||
if project == "dms":
|
||||
fallback = _dms_workspace_pack_dir(pack_name)
|
||||
if fallback is not None:
|
||||
return fallback
|
||||
try:
|
||||
return primary.resolve()
|
||||
except OSError:
|
||||
return primary
|
||||
|
||||
|
||||
def _read_jsonl_tail(path: Path, n: int = 10) -> list[dict]:
|
||||
@@ -111,28 +163,66 @@ def get_pending_report(wf: dict | None = None) -> dict[str, Any]:
|
||||
ingest_log = root / "manifests" / "ingest_log.jsonl"
|
||||
proj["recent_ingest"] = _read_jsonl_tail(ingest_log, 10)
|
||||
|
||||
for task, tcfg in reg.get("tasks", {}).items():
|
||||
proj["task_defs"][task] = {
|
||||
"type": tcfg.get("type"),
|
||||
"nc": tcfg.get("nc"),
|
||||
"names": tcfg.get("names"),
|
||||
"task_dir": tcfg.get("task_dir", task),
|
||||
}
|
||||
inbox_batches: list[str] = []
|
||||
ib = root / "inbox" / task
|
||||
if ib.is_dir():
|
||||
inbox_batches = [
|
||||
d.name for d in ib.iterdir()
|
||||
if d.is_dir() and not d.name.startswith(".")
|
||||
]
|
||||
proj["task_defs"] = task_defs_for_pending(reg)
|
||||
|
||||
sources_pending: dict[str, list[str]] = {}
|
||||
def _scan_task_inbox(task_id: str, mode: str | None) -> list[str]:
|
||||
ib = inbox_dir(root, task_id, mode, reg)
|
||||
if not ib.is_dir():
|
||||
return []
|
||||
subdirs = [
|
||||
d.name for d in ib.iterdir() if d.is_dir() and not d.name.startswith(".")
|
||||
]
|
||||
if subdirs:
|
||||
return subdirs
|
||||
tcfg = reg.get("tasks", {}).get(task_id, {})
|
||||
if tcfg.get("type") == "multi" and mode:
|
||||
if dms_has_images(ib):
|
||||
return [mode]
|
||||
return []
|
||||
|
||||
def _multi_inbox_batch_rows(task: str, mode: str) -> list[dict[str, Any]]:
|
||||
"""multi 任务:inbox 可能即批次根目录(如 inbox/dam/batch_0516)。"""
|
||||
ib = inbox_dir(root, task, mode, reg)
|
||||
if not ib.is_dir():
|
||||
return []
|
||||
rows: list[dict[str, Any]] = []
|
||||
subdirs = [
|
||||
d.name for d in ib.iterdir() if d.is_dir() and not d.name.startswith(".")
|
||||
]
|
||||
if subdirs:
|
||||
for batch_name in subdirs:
|
||||
row = enrich_batch(
|
||||
ib / batch_name,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=None,
|
||||
batch=batch_name,
|
||||
location="inbox",
|
||||
)
|
||||
row["mode"] = mode
|
||||
rows.append(row)
|
||||
elif dms_has_images(ib) or (ib / META_FILENAME).is_file():
|
||||
row = enrich_batch(
|
||||
ib,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=None,
|
||||
batch=mode,
|
||||
location="inbox",
|
||||
)
|
||||
row["mode"] = mode
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
def _scan_sources(task_id: str, mode: str | None) -> dict[str, list[str]]:
|
||||
mcfg = get_mode_config(task_id, mode, reg)
|
||||
pending: dict[str, list[str]] = {}
|
||||
for pack_name in all_names:
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
src_root = pack_dir / tcfg["task_dir"] / src_sub
|
||||
src_root = pack_dir / mcfg["task_dir"] / src_sub
|
||||
if src_root.is_dir():
|
||||
batches = [
|
||||
d.name for d in src_root.iterdir()
|
||||
@@ -141,44 +231,63 @@ def get_pending_report(wf: dict | None = None) -> dict[str, Any]:
|
||||
and not d.name.startswith(".")
|
||||
]
|
||||
if batches:
|
||||
sources_pending[pack_name] = batches
|
||||
pending[pack_name] = batches
|
||||
return pending
|
||||
|
||||
proj["tasks"][task] = {
|
||||
"inbox": inbox_batches,
|
||||
"sources": sources_pending,
|
||||
}
|
||||
|
||||
for batch_name in inbox_batches:
|
||||
batch_dir = ib / batch_name
|
||||
report["batches"].append(
|
||||
enrich_batch(
|
||||
batch_dir,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=None,
|
||||
batch=batch_name,
|
||||
location="inbox",
|
||||
)
|
||||
)
|
||||
|
||||
for pack_name, batch_list in sources_pending.items():
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
src_root = pack_dir / tcfg["task_dir"] / src_sub
|
||||
for batch_name in batch_list:
|
||||
batch_dir = src_root / batch_name
|
||||
for task, tcfg in reg.get("tasks", {}).items():
|
||||
if tcfg.get("type") == "multi":
|
||||
task_inbox: dict[str, list[str]] = {}
|
||||
task_sources: dict[str, dict[str, list[str]]] = {}
|
||||
for mode in (tcfg.get("modes") or {}):
|
||||
task_inbox[mode] = _scan_task_inbox(task, mode)
|
||||
task_sources[mode] = _scan_sources(task, mode)
|
||||
for row in _multi_inbox_batch_rows(task, mode):
|
||||
report["batches"].append(row)
|
||||
for pack_name, batch_list in task_sources[mode].items():
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
src_root = pack_dir / get_mode_config(task, mode, reg)["task_dir"] / src_sub
|
||||
for batch_name in batch_list:
|
||||
row = enrich_batch(
|
||||
src_root / batch_name,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=pack_name,
|
||||
batch=batch_name,
|
||||
location="sources",
|
||||
)
|
||||
row["mode"] = mode
|
||||
report["batches"].append(row)
|
||||
proj["tasks"][task] = {"inbox": task_inbox, "sources": task_sources}
|
||||
else:
|
||||
inbox_batches = _scan_task_inbox(task, None)
|
||||
sources_pending = _scan_sources(task, None)
|
||||
proj["tasks"][task] = {"inbox": inbox_batches, "sources": sources_pending}
|
||||
ib = inbox_dir(root, task, None, reg)
|
||||
for batch_name in inbox_batches:
|
||||
report["batches"].append(
|
||||
enrich_batch(
|
||||
batch_dir,
|
||||
ib / batch_name,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=pack_name,
|
||||
pack=None,
|
||||
batch=batch_name,
|
||||
location="sources",
|
||||
location="inbox",
|
||||
)
|
||||
)
|
||||
for pack_name, batch_list in sources_pending.items():
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
src_root = pack_dir / tcfg["task_dir"] / src_sub
|
||||
for batch_name in batch_list:
|
||||
report["batches"].append(
|
||||
enrich_batch(
|
||||
src_root / batch_name,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=pack_name,
|
||||
batch=batch_name,
|
||||
location="sources",
|
||||
)
|
||||
)
|
||||
|
||||
if pname == "lane":
|
||||
proj["packs"] = {}
|
||||
@@ -297,7 +406,7 @@ def _count_images_in_dir(img_dir: Path) -> int:
|
||||
try:
|
||||
with os.scandir(img_dir) as it:
|
||||
for entry in it:
|
||||
if not entry.is_file(follow_symlinks=False):
|
||||
if not entry.is_file(follow_symlinks=True):
|
||||
continue
|
||||
if Path(entry.name).suffix.lower() in IMAGE_EXTS:
|
||||
total += 1
|
||||
@@ -325,8 +434,8 @@ def _iter_label_files(label_dirs: list[Path]):
|
||||
continue
|
||||
try:
|
||||
with os.scandir(label_dir) as it:
|
||||
stack = [entry.path for entry in it if entry.is_dir(follow_symlinks=False)]
|
||||
files = [entry.path for entry in it if entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt")]
|
||||
stack = [entry.path for entry in it if entry.is_dir(follow_symlinks=True)]
|
||||
files = [entry.path for entry in it if entry.is_file(follow_symlinks=True) and entry.name.endswith(".txt")]
|
||||
except OSError:
|
||||
continue
|
||||
for fp in files:
|
||||
@@ -336,9 +445,9 @@ def _iter_label_files(label_dirs: list[Path]):
|
||||
try:
|
||||
with os.scandir(current) as it:
|
||||
for entry in it:
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
if entry.is_dir(follow_symlinks=True):
|
||||
stack.append(entry.path)
|
||||
elif entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt"):
|
||||
elif entry.is_file(follow_symlinks=True) and entry.name.endswith(".txt"):
|
||||
yield Path(entry.path)
|
||||
except OSError:
|
||||
continue
|
||||
@@ -348,6 +457,49 @@ def _label_dirs_for_task(task_data: Path) -> list[Path]:
|
||||
return [task_data / "labels" / "train", task_data / "labels" / "val", task_data / "labels"]
|
||||
|
||||
|
||||
def _task_data_has_labels(task_data: Path) -> bool:
|
||||
return any(p.is_dir() for p in _label_dirs_for_task(task_data))
|
||||
|
||||
|
||||
def resolve_task_data_for_scan(pack_dir: Path, task_dir: str) -> Path:
|
||||
"""解析可扫描的 task 数据目录。
|
||||
|
||||
DAM 等批次在宿主机上常为指向绝对路径的 symlink,Docker 内 resolve() 会落到
|
||||
容器外路径导致 is_dir() 为 False、bbox 采样为空。此处回映射到 AS_WORKSPACE_ROOT
|
||||
并尝试 pack 级 _dam_stash_* 目录。
|
||||
"""
|
||||
task_data = pack_dir / task_dir
|
||||
if _task_data_has_labels(task_data):
|
||||
return task_data
|
||||
|
||||
if task_data.is_symlink():
|
||||
try:
|
||||
raw = os.readlink(task_data)
|
||||
link = Path(raw)
|
||||
candidates: list[Path] = []
|
||||
if link.is_absolute():
|
||||
s = str(link).replace("\\", "/")
|
||||
for marker in ("DMS/DATASET/", "DMS/DATASET"):
|
||||
idx = s.find(marker)
|
||||
if idx >= 0 and WORKSPACE_ROOT:
|
||||
candidates.append(WORKSPACE_ROOT / s[idx:])
|
||||
else:
|
||||
candidates.append((task_data.parent / link).resolve())
|
||||
for cand in candidates:
|
||||
if _task_data_has_labels(cand):
|
||||
return cand
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
leaf = Path(task_dir).name
|
||||
if leaf.startswith("batch_"):
|
||||
stash = pack_dir / f"_dam_stash_{leaf.split('_', 1)[1]}"
|
||||
if _task_data_has_labels(stash):
|
||||
return stash
|
||||
|
||||
return task_data
|
||||
|
||||
|
||||
def _parse_bbox_wh(parts: list[str]) -> list[float] | None:
|
||||
if len(parts) < 5:
|
||||
return None
|
||||
@@ -574,16 +726,80 @@ def _catalog_signature(wf: dict) -> dict[str, Any]:
|
||||
return build_catalog_signature(wf, proj_root)
|
||||
|
||||
|
||||
def _merge_multi_task_entry(
|
||||
dms: dict[str, Any],
|
||||
task_id: str,
|
||||
tcfg: dict,
|
||||
legacy_modes: dict[str, dict],
|
||||
) -> None:
|
||||
entry = dms.setdefault(task_id, {})
|
||||
entry["type"] = "multi"
|
||||
entry["domain"] = tcfg.get("domain", "dms")
|
||||
entry["label"] = tcfg.get("label", task_id)
|
||||
entry["domain_label"] = DOMAIN_LABELS.get(tcfg.get("domain", "dms"), tcfg.get("domain", "dms"))
|
||||
modes = entry.setdefault("modes", {})
|
||||
for mode, mcfg in (tcfg.get("modes") or {}).items():
|
||||
mode_entry = modes.setdefault(mode, {})
|
||||
mode_entry.setdefault("label", mcfg.get("label", mode))
|
||||
mode_entry.setdefault("type", mcfg.get("type", "detect"))
|
||||
mode_entry.setdefault("nc", mcfg.get("nc"))
|
||||
mode_entry.setdefault("names", mcfg.get("names"))
|
||||
mode_entry.setdefault("packs", [])
|
||||
mode_entry.setdefault("class_counts", {})
|
||||
if mode in legacy_modes:
|
||||
leg = legacy_modes[mode]
|
||||
if not mode_entry.get("packs") and leg.get("packs"):
|
||||
mode_entry["packs"] = leg["packs"]
|
||||
if not mode_entry.get("class_counts") and leg.get("class_counts"):
|
||||
mode_entry["class_counts"] = leg["class_counts"]
|
||||
stray = entry.pop("packs", None)
|
||||
first_mode = next(iter(modes), None)
|
||||
if stray and isinstance(stray, list) and first_mode and not modes[first_mode].get("packs"):
|
||||
modes[first_mode]["packs"] = stray
|
||||
entry.pop("nc", None)
|
||||
entry.pop("names", None)
|
||||
|
||||
|
||||
def _normalize_catalog_dms(dms: dict[str, Any], reg: dict) -> dict[str, Any]:
|
||||
"""修复旧缓存:isa/isa_class→forward;dam_0417→dam;补全 multi 结构。"""
|
||||
if not dms:
|
||||
return dms
|
||||
tasks_cfg = (reg or {}).get("tasks") or {}
|
||||
|
||||
legacy_forward = {}
|
||||
for old_key, mode in (("isa", "detect"), ("isa_class", "classify")):
|
||||
if old_key in dms:
|
||||
legacy_forward[mode] = dms.pop(old_key)
|
||||
fwd_cfg = tasks_cfg.get("forward") or {}
|
||||
if fwd_cfg.get("type") == "multi" or legacy_forward:
|
||||
_merge_multi_task_entry(dms, "forward", fwd_cfg, legacy_forward)
|
||||
|
||||
legacy_dam: dict[str, dict] = {}
|
||||
if "dam_0417" in dms:
|
||||
legacy_dam["batch_0417"] = dms.pop("dam_0417")
|
||||
if "dam" in dms and (tasks_cfg.get("dam") or {}).get("type") == "multi":
|
||||
old_dam = dms.pop("dam")
|
||||
if old_dam.get("modes"):
|
||||
dms["dam"] = old_dam
|
||||
else:
|
||||
legacy_dam["batch_0516"] = old_dam
|
||||
dam_cfg = tasks_cfg.get("dam") or {}
|
||||
if dam_cfg.get("type") == "multi" or legacy_dam:
|
||||
_merge_multi_task_entry(dms, "dam", dam_cfg, legacy_dam)
|
||||
|
||||
return dms
|
||||
|
||||
|
||||
def _build_catalog(wf: dict, *, prefer_reports: bool = True) -> tuple[dict[str, Any], str]:
|
||||
out: dict[str, Any] = {"workspace": str(WORKSPACE), "dms": {}, "lane": {}}
|
||||
build_source = "scan"
|
||||
|
||||
reports = load_dms_reports() if prefer_reports else None
|
||||
reports = load_dms_reports()
|
||||
report_splits: dict[tuple[str, str], dict[str, int]] = {}
|
||||
report_classes: dict[str, dict[str, int]] = {}
|
||||
if reports:
|
||||
report_splits, report_classes = reports
|
||||
build_source = "reports"
|
||||
build_source = "reports" if prefer_reports else "scan+reports"
|
||||
|
||||
root = proj_root(wf, "dms")
|
||||
reg_path = root / wf["projects"]["dms"]["registry"]
|
||||
@@ -620,61 +836,91 @@ def _build_catalog(wf: dict, *, prefer_reports: bool = True) -> tuple[dict[str,
|
||||
if summary_path.is_file():
|
||||
class_by_task = _parse_class_summary(summary_path.read_text(encoding="utf-8"))
|
||||
|
||||
for task, tcfg in reg.get("tasks", {}).items():
|
||||
entry: dict[str, Any] = {
|
||||
"type": tcfg.get("type"),
|
||||
"nc": tcfg.get("nc"),
|
||||
"names": tcfg.get("names"),
|
||||
"class_counts": class_by_task.get(task, {}),
|
||||
"packs": [],
|
||||
"drop_paths": {
|
||||
"inbox": str((root / "inbox" / task).resolve()),
|
||||
"sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "sources" / "<batch>").resolve()),
|
||||
},
|
||||
}
|
||||
for p in packs_reg.get("packs", []):
|
||||
pack_name = p["name"]
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
task_data = pack_dir / tcfg.get("task_dir", task)
|
||||
rep = report_splits.get((task, pack_name))
|
||||
if rep:
|
||||
def _pack_row(task_id: str, mode: str | None, mcfg: dict, pack_name: str, p: dict, task_data: Path) -> dict[str, Any]:
|
||||
rep_key = report_task_key(task_id, mode)
|
||||
rep = report_splits.get((rep_key, pack_name))
|
||||
split_counts = _count_split_images(task_data)
|
||||
scan_total = sum(split_counts.values())
|
||||
if rep and (scan_total == 0 or not _task_data_has_labels(task_data)):
|
||||
split_counts = {"train": rep["train"], "val": rep["val"], "test": rep["test"]}
|
||||
class_counts = report_classes.get(rep_key, class_by_task.get(rep_key, {}))
|
||||
bbox_points = _collect_bbox_points_sample(task_data) if _task_data_has_labels(task_data) else []
|
||||
label_distribution = {
|
||||
"class_counts": class_counts,
|
||||
"label_files": 0,
|
||||
"sampled": True,
|
||||
"total_boxes": sum(class_counts.values()) if class_counts else 0,
|
||||
"bbox_points": bbox_points,
|
||||
}
|
||||
else:
|
||||
label_distribution = _collect_pack_label_distribution(task_data, mcfg)
|
||||
if not label_distribution["class_counts"] and rep_key in report_classes:
|
||||
label_distribution["class_counts"] = report_classes[rep_key]
|
||||
if scan_total == 0 and rep:
|
||||
split_counts = {"train": rep["train"], "val": rep["val"], "test": rep["test"]}
|
||||
class_counts = report_classes.get(task, class_by_task.get(task, {}))
|
||||
bbox_points = _collect_bbox_points_sample(task_data)
|
||||
label_distribution = {
|
||||
"class_counts": class_counts,
|
||||
"label_files": 0,
|
||||
"sampled": True,
|
||||
"total_boxes": sum(class_counts.values()) if class_counts else 0,
|
||||
"bbox_points": bbox_points,
|
||||
}
|
||||
else:
|
||||
split_counts = _count_split_images(task_data)
|
||||
label_distribution = _collect_pack_label_distribution(task_data, tcfg)
|
||||
if not label_distribution["class_counts"] and task in report_classes:
|
||||
label_distribution["class_counts"] = report_classes[task]
|
||||
entry["packs"].append({
|
||||
"name": pack_name,
|
||||
"path": p.get("path"),
|
||||
"role": p.get("role"),
|
||||
"frozen": p.get("frozen", False),
|
||||
"enabled": pack_name in wf["projects"]["dms"].get("active_packs", []),
|
||||
"train_images": split_counts.get("train", 0),
|
||||
"val_images": split_counts.get("val", 0),
|
||||
"test_images": split_counts.get("test", 0),
|
||||
"class_counts": label_distribution["class_counts"],
|
||||
"label_files": label_distribution["label_files"],
|
||||
"total_boxes": label_distribution["total_boxes"],
|
||||
"sampled": label_distribution["sampled"],
|
||||
"bbox_points": label_distribution["bbox_points"],
|
||||
})
|
||||
if not entry["class_counts"] and task in report_classes:
|
||||
entry["class_counts"] = report_classes[task]
|
||||
return {
|
||||
"name": pack_name,
|
||||
"path": p.get("path"),
|
||||
"role": p.get("role"),
|
||||
"frozen": p.get("frozen", False),
|
||||
"enabled": pack_name in wf["projects"]["dms"].get("active_packs", []),
|
||||
"train_images": split_counts.get("train", 0),
|
||||
"val_images": split_counts.get("val", 0),
|
||||
"test_images": split_counts.get("test", 0),
|
||||
"class_counts": label_distribution["class_counts"],
|
||||
"label_files": label_distribution["label_files"],
|
||||
"total_boxes": label_distribution["total_boxes"],
|
||||
"sampled": label_distribution["sampled"],
|
||||
"bbox_points": label_distribution["bbox_points"],
|
||||
}
|
||||
|
||||
for task, entry in iter_catalog_tasks(reg):
|
||||
tcfg = reg["tasks"][task]
|
||||
if tcfg.get("type") == "multi":
|
||||
entry["drop_paths"] = {
|
||||
**{
|
||||
f"inbox_{mode}": str(inbox_dir(root, task, mode, reg).resolve())
|
||||
for mode in (tcfg.get("modes") or {})
|
||||
},
|
||||
"sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "<subdir>" / "sources" / "<batch>").resolve()),
|
||||
}
|
||||
for mode in (tcfg.get("modes") or {}):
|
||||
mcfg = get_mode_config(task, mode, reg)
|
||||
mode_entry = entry["modes"][mode]
|
||||
rep_key = report_task_key(task, mode)
|
||||
mode_entry["class_counts"] = class_by_task.get("isa_detect", class_by_task.get(rep_key, {})) if mode == "detect" else class_by_task.get("isa_class_0116", class_by_task.get(rep_key, {}))
|
||||
if not mode_entry["class_counts"] and rep_key in report_classes:
|
||||
mode_entry["class_counts"] = report_classes[rep_key]
|
||||
for p in packs_reg.get("packs", []):
|
||||
pack_name = p["name"]
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
task_data = resolve_task_data_for_scan(pack_dir, mcfg["task_dir"])
|
||||
mode_entry["packs"].append(_pack_row(task, mode, mcfg, pack_name, p, task_data))
|
||||
else:
|
||||
entry["drop_paths"] = {
|
||||
"inbox": str(inbox_dir(root, task, None, reg).resolve()),
|
||||
"sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "sources" / "<batch>").resolve()),
|
||||
}
|
||||
rep_key = task
|
||||
if not entry["class_counts"]:
|
||||
entry["class_counts"] = class_by_task.get(task, report_classes.get(task, {}))
|
||||
for p in packs_reg.get("packs", []):
|
||||
pack_name = p["name"]
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
task_data = resolve_task_data_for_scan(pack_dir, tcfg.get("task_dir", task))
|
||||
entry["packs"].append(_pack_row(task, None, tcfg, pack_name, p, task_data))
|
||||
if not entry["class_counts"] and task in report_classes:
|
||||
entry["class_counts"] = report_classes[task]
|
||||
out["dms"][task] = entry
|
||||
|
||||
out["dms"] = _normalize_catalog_dms(out["dms"], reg)
|
||||
|
||||
root = proj_root(wf, "lane")
|
||||
try:
|
||||
reg = load_pack_registry("lane", root, wf)
|
||||
@@ -718,6 +964,12 @@ def get_catalog(
|
||||
full_catalog, build_source = _build_catalog(wf, prefer_reports=not refresh)
|
||||
store_catalog_cache(sig, full_catalog, build_source=build_source)
|
||||
cache_meta = {"cached": False, "build_source": build_source}
|
||||
else:
|
||||
reg_path = proj_root(wf, "dms") / wf["projects"]["dms"]["registry"]
|
||||
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8")) if reg_path.is_file() else {}
|
||||
dms = _normalize_catalog_dms(full_catalog.get("dms") or {}, reg)
|
||||
if dms != full_catalog.get("dms"):
|
||||
full_catalog = {**full_catalog, "dms": dms}
|
||||
|
||||
result: dict[str, Any]
|
||||
if project == "dms" and task_or_pack:
|
||||
|
||||
91
platform/as_platform/data/ingest/dms_inbox_raw.py
Normal file
91
platform/as_platform/data/ingest/dms_inbox_raw.py
Normal file
@@ -0,0 +1,91 @@
|
||||
"""DMS inbox 原图批次(如 addw/ddaw:仅有 images/,待送标,无 YOLO/COCO 标注)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from as_platform.data.batch import count_images, dms_has_images
|
||||
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
|
||||
|
||||
|
||||
def _count_images(path: Path) -> int:
|
||||
return count_images(path)
|
||||
|
||||
|
||||
def _split_image_counts(root: Path) -> dict[str, int]:
|
||||
train = _count_images(root / "images" / "train")
|
||||
val = _count_images(root / "images" / "val")
|
||||
test = _count_images(root / "images" / "test")
|
||||
if train + val + test == 0:
|
||||
train = _count_images(root / "images")
|
||||
if train + val + test == 0 and root.name == "images":
|
||||
train = _count_images(root / "train")
|
||||
val = _count_images(root / "val")
|
||||
test = _count_images(root / "test")
|
||||
if train + val + test == 0:
|
||||
train = _count_images(root)
|
||||
if train + val + test == 0 and (root / "train").is_dir():
|
||||
train = _count_images(root / "train")
|
||||
val = _count_images(root / "val")
|
||||
test = _count_images(root / "test")
|
||||
return {"train": train, "val": val, "test": test}
|
||||
|
||||
|
||||
def _has_label_txts(root: Path) -> bool:
|
||||
for sub in ("labels", "labels/train"):
|
||||
d = root / sub
|
||||
if d.is_dir() and any(d.rglob("*.txt")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_inbox_raw_layout(root: Path) -> bool:
|
||||
if _has_label_txts(root):
|
||||
return False
|
||||
if dms_has_images(root):
|
||||
return True
|
||||
if root.name == "images" and (_count_images(root / "train") > 0 or _count_images(root) > 0):
|
||||
return True
|
||||
if (root / "train").is_dir() and _count_images(root / "train") > 0:
|
||||
return True
|
||||
if _count_images(root) > 0 and not (root / "images").is_dir() and not (root / "labels").is_dir():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class DmsInboxRawAdapter(IngestAdapter):
|
||||
format_id = "dms_inbox_raw"
|
||||
projects = ("dms",)
|
||||
|
||||
def can_handle(self, ctx: IngestContext) -> bool:
|
||||
return _is_inbox_raw_layout(ctx.source_path)
|
||||
|
||||
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
|
||||
root = ctx.source_path
|
||||
split_counts = _split_image_counts(root)
|
||||
sample_count = sum(split_counts.values())
|
||||
warnings: list[str] = []
|
||||
if sample_count == 0:
|
||||
warnings.append(
|
||||
"未找到图片;请填批次根目录(含 images/train/)或 images/、train/ 目录"
|
||||
)
|
||||
|
||||
artifacts: list[str] = []
|
||||
if (root / "images").is_dir():
|
||||
artifacts.append("images/")
|
||||
elif (root / "train").is_dir():
|
||||
artifacts.append("train/(将规范为 images/train)")
|
||||
elif root.name == "images":
|
||||
artifacts.append("images/")
|
||||
|
||||
return NormalizedDataset(
|
||||
format_id=self.format_id,
|
||||
project=ctx.project,
|
||||
task=ctx.task,
|
||||
source_path=str(root),
|
||||
split_counts=split_counts,
|
||||
sample_count=sample_count,
|
||||
annotation_count=0,
|
||||
artifacts=artifacts,
|
||||
warnings=warnings,
|
||||
extra={"stage_hint": "raw_pool"},
|
||||
)
|
||||
@@ -5,6 +5,7 @@ from pathlib import Path
|
||||
|
||||
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
|
||||
from as_platform.data.ingest.dms_coco import DmsCocoAdapter
|
||||
from as_platform.data.ingest.dms_inbox_raw import DmsInboxRawAdapter
|
||||
from as_platform.data.ingest.dms_yolo import DmsYoloAdapter
|
||||
from as_platform.data.ingest.lane_lines import LaneLinesAdapter
|
||||
from as_platform.data.ingest.lane_mask import LaneMaskAdapter
|
||||
@@ -17,6 +18,7 @@ class UnknownFormatError(ValueError):
|
||||
ADAPTERS: tuple[IngestAdapter, ...] = (
|
||||
DmsYoloAdapter(),
|
||||
DmsCocoAdapter(),
|
||||
DmsInboxRawAdapter(),
|
||||
LaneMaskAdapter(),
|
||||
LaneLinesAdapter(),
|
||||
)
|
||||
@@ -32,9 +34,15 @@ def detect_adapter(ctx: IngestContext) -> IngestAdapter:
|
||||
continue
|
||||
if adapter.can_handle(ctx):
|
||||
return adapter
|
||||
hint = ""
|
||||
if ctx.project == "dms":
|
||||
hint = (
|
||||
";DMS 送标/inbox 请使用批次根目录,且至少包含 images/train/*.jpg"
|
||||
"(或已标注的 images/+labels/、COCO annotations/)"
|
||||
)
|
||||
raise UnknownFormatError(
|
||||
f"unable to detect format for project={ctx.project}, task={ctx.task}, "
|
||||
f"source={ctx.source_path}. supported={available_formats(ctx.project)}"
|
||||
f"source={ctx.source_path}. supported={available_formats(ctx.project)}{hint}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -8,10 +8,14 @@ import uuid
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO
|
||||
from typing import Any, BinaryIO
|
||||
|
||||
import yaml
|
||||
|
||||
from as_platform.config import MANIFESTS
|
||||
from as_platform.data.batch import META_FILENAME, dms_has_images, enrich_batch, write_meta
|
||||
from as_platform.data.catalog_cache import invalidate_catalog_cache
|
||||
from as_platform.data.core import load_wf, proj_root
|
||||
from as_platform.data.ingest import inspect_uploaded_dataset
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import DatasetCandidate
|
||||
@@ -37,6 +41,7 @@ def create_uploaded_candidate(
|
||||
*,
|
||||
project: str,
|
||||
task: str | None,
|
||||
mode: str | None = None,
|
||||
original_name: str,
|
||||
upload_size_bytes: int,
|
||||
submitted_by_name: str | None,
|
||||
@@ -51,6 +56,7 @@ def create_uploaded_candidate(
|
||||
id=candidate_id,
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
status="uploaded",
|
||||
source_type="upload",
|
||||
original_name=original_name,
|
||||
@@ -82,15 +88,17 @@ def write_candidate_upload(candidate_id: str, stream: BinaryIO, chunk_size: int
|
||||
return str(path)
|
||||
|
||||
|
||||
def list_candidates(limit: int = 100) -> list[dict]:
|
||||
def list_candidates(*, offset: int = 0, limit: int = 20) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
rows = (
|
||||
db.query(DatasetCandidate)
|
||||
.order_by(DatasetCandidate.created_at.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return [r.to_dict() for r in rows]
|
||||
q = db.query(DatasetCandidate).order_by(DatasetCandidate.created_at.desc())
|
||||
total = q.count()
|
||||
rows = q.offset(max(0, offset)).limit(max(1, limit)).all()
|
||||
return {
|
||||
"items": [r.to_dict() for r in rows],
|
||||
"total": total,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
|
||||
def get_candidate(candidate_id: str) -> dict | None:
|
||||
@@ -153,6 +161,8 @@ def analyze_uploaded_candidate(candidate_id: str) -> dict:
|
||||
|
||||
try:
|
||||
dataset_root = _extract_to_staging(upload_path, staging_dir)
|
||||
if project == "dms":
|
||||
dataset_root = _ensure_dms_inbox_layout(dataset_root)
|
||||
normalized = inspect_uploaded_dataset(project, task, dataset_root)
|
||||
report_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_file.write_text(json.dumps(normalized.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
@@ -177,3 +187,306 @@ def analyze_uploaded_candidate(candidate_id: str) -> dict:
|
||||
rec.error_message = str(e)
|
||||
db.flush()
|
||||
raise
|
||||
|
||||
|
||||
_SPLIT_DIR_NAMES = frozenset({"train", "val", "test"})
|
||||
_STRUCTURE_DIR_NAMES = frozenset({"images", "labels", "annotations"})
|
||||
|
||||
|
||||
def _dataset_root_from_dir(source_dir: Path) -> Path:
|
||||
"""解析批次根目录;勿把 images/ 或 train/ 误剥成更深层导致丢失 images 层。"""
|
||||
if not source_dir.is_dir():
|
||||
raise FileNotFoundError(f"not a directory: {source_dir}")
|
||||
subdirs = [p for p in source_dir.iterdir() if p.is_dir() and not p.name.startswith(".")]
|
||||
files = [p for p in source_dir.iterdir() if p.is_file()]
|
||||
if len(subdirs) == 1 and not files:
|
||||
only = subdirs[0]
|
||||
if only.name in _SPLIT_DIR_NAMES or only.name in _STRUCTURE_DIR_NAMES:
|
||||
return source_dir
|
||||
return only
|
||||
return source_dir
|
||||
|
||||
|
||||
def _ensure_dms_inbox_layout(root: Path) -> Path:
|
||||
"""将 dataset/train 布局规范为 …/images/train;已是批次根或 images/ 目录则不改动。"""
|
||||
if not root.is_dir():
|
||||
return root
|
||||
from as_platform.data.batch import count_images, dms_has_images
|
||||
|
||||
if dms_has_images(root):
|
||||
return root
|
||||
if root.name == "images" and count_images(root / "train") > 0:
|
||||
return root
|
||||
if count_images(root) > 0 and not any(
|
||||
(root / sub).is_dir() for sub in ("images", "train", "labels")
|
||||
):
|
||||
images_dir = root / "images"
|
||||
images_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest_train = images_dir / "train"
|
||||
dest_train.mkdir(parents=True, exist_ok=True)
|
||||
for item in list(root.iterdir()):
|
||||
if item.is_file() and item.suffix.lower() in {".jpg", ".jpeg", ".png", ".bmp", ".webp"}:
|
||||
shutil.move(str(item), str(dest_train / item.name))
|
||||
return root
|
||||
|
||||
train_dir = root / "train"
|
||||
if not train_dir.is_dir() or count_images(train_dir) == 0:
|
||||
return root
|
||||
images_dir = root / "images"
|
||||
if images_dir.is_dir() and count_images(images_dir / "train") > 0:
|
||||
return root
|
||||
images_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest_train = images_dir / "train"
|
||||
if dest_train.exists():
|
||||
shutil.rmtree(dest_train, ignore_errors=True)
|
||||
shutil.move(str(train_dir), str(dest_train))
|
||||
return root
|
||||
|
||||
|
||||
def analyze_directory_candidate(candidate_id: str, source_dir: Path | None = None) -> dict:
|
||||
"""分析目录型数据源(飞书 data_path / NAS),无需 zip 上传。"""
|
||||
upload_dir, staging_dir, report_file = _candidate_dirs(candidate_id)
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if not rec:
|
||||
raise ValueError(f"candidate not found: {candidate_id}")
|
||||
rec.status = "analyzing"
|
||||
rec.error_message = None
|
||||
db.flush()
|
||||
project = rec.project
|
||||
task = rec.task
|
||||
root = source_dir or Path(rec.upload_path)
|
||||
|
||||
if not root.is_dir():
|
||||
msg = f"source directory missing: {root}"
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if rec:
|
||||
rec.status = "failed"
|
||||
rec.error_message = msg
|
||||
raise FileNotFoundError(msg)
|
||||
|
||||
try:
|
||||
dataset_root = _dataset_root_from_dir(root)
|
||||
if staging_dir.exists():
|
||||
shutil.rmtree(staging_dir)
|
||||
shutil.copytree(dataset_root, staging_dir / "dataset", dirs_exist_ok=True)
|
||||
analyzed_root = staging_dir / "dataset"
|
||||
if project == "dms":
|
||||
analyzed_root = _ensure_dms_inbox_layout(analyzed_root)
|
||||
normalized = inspect_uploaded_dataset(project, task, analyzed_root)
|
||||
report_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_file.write_text(json.dumps(normalized.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if not rec:
|
||||
raise ValueError(f"candidate not found during finalize: {candidate_id}")
|
||||
rec.status = "analyzed"
|
||||
rec.analyzed_source_path = str(analyzed_root)
|
||||
rec.format_id = normalized.format_id
|
||||
rec.set_split_counts(normalized.split_counts)
|
||||
rec.set_quality(normalized.to_dict())
|
||||
rec.error_message = None
|
||||
db.flush()
|
||||
invalidate_catalog_cache()
|
||||
return normalized.to_dict()
|
||||
except Exception as e:
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if rec:
|
||||
rec.status = "failed"
|
||||
rec.error_message = str(e)
|
||||
db.flush()
|
||||
raise
|
||||
|
||||
|
||||
def create_directory_candidate(
|
||||
*,
|
||||
project: str,
|
||||
task: str | None,
|
||||
mode: str | None,
|
||||
source_dir: Path,
|
||||
source_type: str = "platform_delivery",
|
||||
external_id: str | None = None,
|
||||
feishu_record_id: str | None = None,
|
||||
) -> dict:
|
||||
"""为外部目录(平台送标申请 / 飞书台账)创建入湖候选。"""
|
||||
candidate_id = _new_candidate_id()
|
||||
upload_dir, _, _ = _candidate_dirs(candidate_id)
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
with session_scope() as db:
|
||||
rec = DatasetCandidate(
|
||||
id=candidate_id,
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
status="uploaded",
|
||||
source_type=source_type,
|
||||
original_name=source_dir.name,
|
||||
upload_path=str(source_dir.resolve()),
|
||||
upload_size_bytes=0,
|
||||
external_id=external_id,
|
||||
feishu_record_id=feishu_record_id,
|
||||
)
|
||||
db.add(rec)
|
||||
db.flush()
|
||||
return rec.to_dict()
|
||||
|
||||
|
||||
def create_feishu_directory_candidate(
|
||||
*,
|
||||
project: str,
|
||||
task: str | None,
|
||||
mode: str | None,
|
||||
source_dir: Path,
|
||||
external_id: str | None = None,
|
||||
feishu_record_id: str | None = None,
|
||||
) -> dict:
|
||||
"""为飞书台账行创建候选并指向 NAS/本地目录。"""
|
||||
return create_directory_candidate(
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
source_dir=source_dir,
|
||||
source_type="feishu_bitable",
|
||||
external_id=external_id,
|
||||
feishu_record_id=feishu_record_id,
|
||||
)
|
||||
|
||||
|
||||
def _copy_tree_into(dest: Path, src: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for item in src.iterdir():
|
||||
target = dest / item.name
|
||||
if item.is_dir():
|
||||
if target.exists():
|
||||
_copy_tree_into(target, item)
|
||||
else:
|
||||
shutil.copytree(item, target)
|
||||
else:
|
||||
shutil.copy2(item, target)
|
||||
|
||||
|
||||
def _resolve_dms_inbox_dest(root: Path, reg: dict, task: str, mode: str | None, batch_name: str) -> Path:
|
||||
import sys
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
|
||||
scripts = WORKSPACE / "datasets" / "dms" / "scripts"
|
||||
if str(scripts) not in sys.path:
|
||||
sys.path.insert(0, str(scripts))
|
||||
from task_registry import inbox_dir, resolve_task_id
|
||||
|
||||
task_r, mode_r = resolve_task_id(task, mode)
|
||||
tcfg = (reg.get("tasks") or {}).get(task_r) or {}
|
||||
ib = inbox_dir(root, task_r, mode_r, reg)
|
||||
if tcfg.get("type") == "multi" and mode_r:
|
||||
if dms_has_images(ib) or (ib / META_FILENAME).is_file():
|
||||
return ib
|
||||
return ib / batch_name
|
||||
return ib / batch_name
|
||||
|
||||
|
||||
def promote_candidate_to_inbox(
|
||||
candidate_id: str,
|
||||
*,
|
||||
batch: str | None = None,
|
||||
mode: str | None = None,
|
||||
) -> dict:
|
||||
"""将 analyzed 候选数据复制到 inbox,并登记 batch.meta。"""
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if not rec:
|
||||
raise ValueError(f"candidate not found: {candidate_id}")
|
||||
if rec.status not in ("analyzed",):
|
||||
raise ValueError(f"candidate 状态须为 analyzed,当前: {rec.status}")
|
||||
if not rec.analyzed_source_path:
|
||||
raise ValueError("缺少 analyzed_source_path,请先完成分析")
|
||||
project = rec.project
|
||||
task = rec.task
|
||||
eff_mode = mode or rec.mode
|
||||
src = Path(rec.analyzed_source_path)
|
||||
cand_format_id = rec.format_id
|
||||
|
||||
if not src.is_dir():
|
||||
raise FileNotFoundError(f"分析目录不存在: {src}")
|
||||
|
||||
wf = load_wf()
|
||||
root = proj_root(wf, project)
|
||||
batch_name = batch or src.name or candidate_id.split("-", 1)[-1]
|
||||
|
||||
if project == "dms":
|
||||
if not task:
|
||||
raise ValueError("DMS 晋级需要 task")
|
||||
reg_path = root / wf["projects"]["dms"]["registry"]
|
||||
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
|
||||
tcfg = (reg.get("tasks") or {}).get(task) or {}
|
||||
if tcfg.get("type") == "multi" and not eff_mode:
|
||||
raise ValueError(f"任务 {task} 为 multi,须指定 mode(如 batch_0516)")
|
||||
dest = _resolve_dms_inbox_dest(root, reg, task, eff_mode, batch_name)
|
||||
reg_batch = eff_mode or batch_name
|
||||
else:
|
||||
dest = root / "inbox" / batch_name
|
||||
reg_batch = batch_name
|
||||
|
||||
if dest.exists() and any(dest.iterdir()):
|
||||
_copy_tree_into(dest, src)
|
||||
else:
|
||||
if dest.exists():
|
||||
shutil.rmtree(dest)
|
||||
shutil.copytree(src, dest)
|
||||
|
||||
row = enrich_batch(
|
||||
dest,
|
||||
project=project,
|
||||
task=task if project == "dms" else None,
|
||||
pack=None,
|
||||
batch=reg_batch,
|
||||
location="inbox",
|
||||
)
|
||||
fmt = row.get("format", "yolo")
|
||||
if cand_format_id == "dms_inbox_raw":
|
||||
fmt = "inbox_raw"
|
||||
meta_payload = {
|
||||
"schema": "huaxu-batch-v1",
|
||||
"project": project,
|
||||
"task": task,
|
||||
"batch": reg_batch,
|
||||
"stage": "raw_pool",
|
||||
"location": "inbox",
|
||||
"format": fmt,
|
||||
"counts": row.get("counts", {}),
|
||||
}
|
||||
if eff_mode:
|
||||
meta_payload["mode"] = eff_mode
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if rec:
|
||||
if rec.external_id:
|
||||
meta_payload["external_id"] = rec.external_id
|
||||
if rec.feishu_record_id:
|
||||
meta_payload["feishu_record_id"] = rec.feishu_record_id
|
||||
if rec.source_type and rec.source_type != "upload":
|
||||
meta_payload["source_type"] = rec.source_type
|
||||
write_meta(dest, meta_payload)
|
||||
meta = {**row, "stage": "raw_pool"}
|
||||
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if rec:
|
||||
rec.status = "promoted"
|
||||
rec.inbox_path = str(dest)
|
||||
rec.promoted_batch = reg_batch
|
||||
if eff_mode:
|
||||
rec.mode = eff_mode
|
||||
db.flush()
|
||||
|
||||
invalidate_catalog_cache()
|
||||
return {
|
||||
"ok": True,
|
||||
"candidate_id": candidate_id,
|
||||
"inbox_path": str(dest),
|
||||
"batch": meta.get("batch", reg_batch),
|
||||
"stage": meta.get("stage"),
|
||||
"counts": meta.get("counts"),
|
||||
}
|
||||
|
||||
179
platform/as_platform/data/simulate.py
Normal file
179
platform/as_platform/data/simulate.py
Normal file
@@ -0,0 +1,179 @@
|
||||
"""世界模型仿真数据生成 — API 接口层。
|
||||
|
||||
实际生成逻辑由外部世界模型引擎完成(通过 subprocess / HTTP 调用)。
|
||||
本模块提供:任务提交、队列管理、状态追踪、结果入库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
from as_platform.db.engine import session_scope
|
||||
|
||||
SIM_JOBS_DIR = WORKSPACE / "manifests" / "simulation_jobs"
|
||||
SIM_OUTPUT_ROOT = WORKSPACE / "datasets" / "dms" / "simulated"
|
||||
|
||||
SCENE_TEMPLATES = {
|
||||
"urban_highway": {"label": "城市快速路", "desc": "多车道高速公路场景"},
|
||||
"urban_street": {"label": "城市街道", "desc": "有交通灯和路口的城市道路"},
|
||||
"rural_road": {"label": "乡村道路", "desc": "双车道乡村公路"},
|
||||
"tunnel": {"label": "隧道", "desc": "隧道内光照变化场景"},
|
||||
"night_city": {"label": "夜间城市", "desc": "低光照城市道路"},
|
||||
"rain_highway": {"label": "雨天高速", "desc": "雨天湿滑路面"},
|
||||
"fog_rural": {"label": "雾天乡村", "desc": "大雾低能见度"},
|
||||
}
|
||||
|
||||
CAMERA_PRESETS = {
|
||||
"truck_front": {"label": "卡车前视", "height": 2.5, "fov": 75, "pitch": -5},
|
||||
"truck_side": {"label": "卡车侧视", "height": 2.5, "fov": 100, "pitch": 0},
|
||||
"car_front": {"label": "轿车前视", "height": 1.2, "fov": 60, "pitch": -3},
|
||||
"car_wide": {"label": "轿车广角", "height": 1.2, "fov": 120, "pitch": 0},
|
||||
}
|
||||
|
||||
OBJECT_CLASSES = ["Pedestrain", "Car", "Truck", "Bus", "Motor-vehicles", "Tricycle", "cones"]
|
||||
|
||||
|
||||
def list_jobs(offset: int = 0, limit: int = 20) -> dict[str, Any]:
|
||||
SIM_JOBS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
jobs = []
|
||||
for f in sorted(SIM_JOBS_DIR.glob("*.json"), reverse=True):
|
||||
try:
|
||||
data = json.loads(f.read_text())
|
||||
data["_id"] = f.stem
|
||||
if data.get("status") != "archived":
|
||||
jobs.append(data)
|
||||
except Exception:
|
||||
pass
|
||||
total = len(jobs)
|
||||
return {"items": jobs[offset:offset + limit], "total": total}
|
||||
|
||||
|
||||
def submit_job(params: dict[str, Any], user_name: str = "") -> dict[str, Any]:
|
||||
job_id = f"sim-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
scene = params.get("scene", "urban_highway")
|
||||
camera = params.get("camera", "truck_front")
|
||||
weather = params.get("weather", "clear")
|
||||
objects = params.get("objects", OBJECT_CLASSES[:4])
|
||||
density = params.get("density", "medium")
|
||||
count = min(params.get("count", 100), 5000)
|
||||
note = params.get("note", "")
|
||||
fov_variant = params.get("fov_variant", False) # 是否生成多FOV变体
|
||||
|
||||
scene_info = SCENE_TEMPLATES.get(scene, {"label": scene})
|
||||
cam_info = CAMERA_PRESETS.get(camera, {"label": camera})
|
||||
|
||||
job = {
|
||||
"id": job_id,
|
||||
"status": "queued",
|
||||
"created_at": now,
|
||||
"submitted_by": user_name,
|
||||
"params": {
|
||||
"scene": scene,
|
||||
"scene_label": scene_info["label"],
|
||||
"camera": camera,
|
||||
"camera_label": cam_info["label"],
|
||||
"camera_height": cam_info.get("height", 2.5),
|
||||
"camera_fov": cam_info.get("fov", 75),
|
||||
"weather": weather,
|
||||
"objects": objects,
|
||||
"density": density,
|
||||
"count": count,
|
||||
"fov_variant": fov_variant,
|
||||
"note": note,
|
||||
},
|
||||
"result": None,
|
||||
"batch_registered": False,
|
||||
}
|
||||
|
||||
SIM_JOBS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(SIM_JOBS_DIR / f"{job_id}.json").write_text(json.dumps(job, ensure_ascii=False, indent=2))
|
||||
|
||||
# Queue actual generation (mock for now — replace with real world model call)
|
||||
_trigger_generation(job_id)
|
||||
|
||||
return job
|
||||
|
||||
|
||||
def get_job(job_id: str) -> dict[str, Any] | None:
|
||||
f = SIM_JOBS_DIR / f"{job_id}.json"
|
||||
if not f.is_file():
|
||||
return None
|
||||
data = json.loads(f.read_text())
|
||||
data["_id"] = f.stem
|
||||
return data
|
||||
|
||||
|
||||
def get_job_images(job_id: str, offset: int = 0, limit: int = 60) -> dict[str, Any]:
|
||||
out_dir = SIM_OUTPUT_ROOT / job_id / "images"
|
||||
if not out_dir.is_dir():
|
||||
return {"items": [], "total": 0}
|
||||
imgs = sorted(out_dir.glob("*.jpg")) + sorted(out_dir.glob("*.png"))
|
||||
total = len(imgs)
|
||||
page = imgs[offset:offset + limit]
|
||||
return {
|
||||
"items": [{"name": p.name, "path": str(p.relative_to(SIM_OUTPUT_ROOT))} for p in page],
|
||||
"total": total,
|
||||
}
|
||||
|
||||
|
||||
def ingest_job_to_batch(job_id: str, task: str = "adas", user_name: str = "") -> dict[str, Any]:
|
||||
"""将仿真生成的数据注册为批次,直接入库。"""
|
||||
job = get_job(job_id)
|
||||
if not job:
|
||||
return {"ok": False, "error": "Job not found"}
|
||||
|
||||
out_dir = SIM_OUTPUT_ROOT / job_id
|
||||
if not out_dir.is_dir():
|
||||
return {"ok": False, "error": "生成数据不存在"}
|
||||
|
||||
# Register as batch
|
||||
from as_platform.data.core import register_batch
|
||||
batch_name = f"sim_{job_id}"
|
||||
try:
|
||||
register_batch(None, "dms", task, batch_name, stage="returned", location="inbox")
|
||||
# Update job status
|
||||
job["status"] = "ingested"
|
||||
job["batch_registered"] = True
|
||||
job["batch_name"] = batch_name
|
||||
(SIM_JOBS_DIR / f"{job_id}.json").write_text(json.dumps(job, ensure_ascii=False, indent=2))
|
||||
return {"ok": True, "batch": batch_name, "task": task}
|
||||
except Exception as e:
|
||||
return {"ok": False, "error": str(e)}
|
||||
|
||||
|
||||
def _trigger_generation(job_id: str) -> None:
|
||||
"""触发实际生成(当前为 mock)。替换为真实世界模型调用。"""
|
||||
import threading
|
||||
|
||||
def _run():
|
||||
try:
|
||||
# TODO: 替换为真实世界模型调用
|
||||
# 例如: subprocess.run(["python", "world_model/generate.py", "--job", job_id])
|
||||
_update_status(job_id, "running")
|
||||
# Mock: create output directory but no actual images
|
||||
out_dir = SIM_OUTPUT_ROOT / job_id / "images"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
# Mock 完成
|
||||
_update_status(job_id, "completed")
|
||||
except Exception as e:
|
||||
_update_status(job_id, "failed", str(e))
|
||||
|
||||
threading.Thread(target=_run, daemon=True, name=f"sim-{job_id}").start()
|
||||
|
||||
|
||||
def _update_status(job_id: str, status: str, error: str = "") -> None:
|
||||
f = SIM_JOBS_DIR / f"{job_id}.json"
|
||||
if f.is_file():
|
||||
data = json.loads(f.read_text())
|
||||
data["status"] = status
|
||||
if error:
|
||||
data["error"] = error
|
||||
if status == "completed":
|
||||
data["completed_at"] = datetime.now(timezone.utc).isoformat()
|
||||
f.write_text(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
161
platform/as_platform/data/versions.py
Normal file
161
platform/as_platform/data/versions.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""数据集版本管理 — snapshot / diff / lineage。版本以 JSON 文件存储在 datasets/<project>/versions/。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
from as_platform.data.core import get_catalog
|
||||
|
||||
|
||||
def _versions_dir(project: str) -> Path:
|
||||
return WORKSPACE / "datasets" / project / "versions"
|
||||
|
||||
|
||||
def list_versions(project: str = "dms") -> list[dict[str, Any]]:
|
||||
"""列出所有版本,按时间倒序。"""
|
||||
vdir = _versions_dir(project)
|
||||
if not vdir.is_dir():
|
||||
return []
|
||||
versions: list[dict[str, Any]] = []
|
||||
for f in sorted(vdir.glob("*.json"), reverse=True):
|
||||
try:
|
||||
data = json.loads(f.read_text(encoding="utf-8"))
|
||||
data["_id"] = f.stem
|
||||
versions.append(data)
|
||||
except Exception:
|
||||
pass
|
||||
return versions
|
||||
|
||||
|
||||
def get_version(project: str, version_id: str) -> dict[str, Any] | None:
|
||||
vf = _versions_dir(project) / f"{version_id}.json"
|
||||
if not vf.is_file():
|
||||
return None
|
||||
data = json.loads(vf.read_text(encoding="utf-8"))
|
||||
data["_id"] = vf.stem
|
||||
return data
|
||||
|
||||
|
||||
def create_snapshot(project: str, description: str = "", author: str = "") -> dict[str, Any]:
|
||||
"""基于当前 catalog 创建数据集快照。"""
|
||||
catalog = get_catalog(refresh=True)
|
||||
dms = catalog.get("dms", {})
|
||||
lane = catalog.get("lane", {})
|
||||
|
||||
# Collect pack summary from catalog
|
||||
packs_summary: dict[str, dict[str, Any]] = {}
|
||||
all_batches: list[str] = []
|
||||
total_images = 0
|
||||
total_labels = 0
|
||||
|
||||
if project == "dms":
|
||||
for task_id, entry in dms.items():
|
||||
entry_d = entry if isinstance(entry, dict) else {}
|
||||
for p in entry_d.get("packs", []) or []:
|
||||
pname = p.get("name", "")
|
||||
if pname not in packs_summary:
|
||||
packs_summary[pname] = {
|
||||
"train_images": 0, "val_images": 0, "test_images": 0,
|
||||
"class_counts": {}, "tasks": [],
|
||||
}
|
||||
s = packs_summary[pname]
|
||||
s["train_images"] += p.get("train_images", 0) or 0
|
||||
s["val_images"] += p.get("val_images", 0) or 0
|
||||
s["test_images"] += p.get("test_images", 0) or 0
|
||||
if task_id not in s["tasks"]:
|
||||
s["tasks"].append(task_id)
|
||||
total_images += (p.get("train_images", 0) or 0) + (p.get("val_images", 0) or 0)
|
||||
total_labels += p.get("label_files", 0) or 0
|
||||
|
||||
# Collect batch names from pending report
|
||||
from as_platform.data.core import get_pending_report
|
||||
report = get_pending_report()
|
||||
for b in report.get("batches", []) or []:
|
||||
if isinstance(b, dict) and b.get("project") == "dms":
|
||||
all_batches.append(b.get("batch", ""))
|
||||
|
||||
# Determine version number
|
||||
existing = list_versions(project)
|
||||
version_num = len(existing) + 1
|
||||
version_id = f"v{version_num}"
|
||||
|
||||
# Get parent version
|
||||
parent_id = existing[0]["_id"] if existing else None
|
||||
|
||||
# Compute diff if parent exists
|
||||
diff: dict[str, Any] | None = None
|
||||
if parent_id and existing:
|
||||
parent = existing[0]
|
||||
parent_packs = set((parent.get("packs") or {}).keys())
|
||||
current_packs = set(packs_summary.keys())
|
||||
parent_batches = set(parent.get("batches", []) or [])
|
||||
current_batches = set(all_batches)
|
||||
diff = {
|
||||
"added_packs": sorted(current_packs - parent_packs),
|
||||
"removed_packs": sorted(parent_packs - current_packs),
|
||||
"added_batches": sorted(current_batches - parent_batches),
|
||||
"removed_batches": sorted(parent_batches - current_batches),
|
||||
}
|
||||
|
||||
snapshot = {
|
||||
"version_id": version_id,
|
||||
"project": project,
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"description": description,
|
||||
"author": author,
|
||||
"parent_version": parent_id,
|
||||
"summary": {
|
||||
"packs_count": len(packs_summary),
|
||||
"total_images": total_images,
|
||||
"total_labels": total_labels,
|
||||
"batches_count": len(all_batches),
|
||||
},
|
||||
"packs": packs_summary,
|
||||
"batches": sorted(all_batches),
|
||||
"diff": diff,
|
||||
}
|
||||
|
||||
# Save to file
|
||||
vdir = _versions_dir(project)
|
||||
vdir.mkdir(parents=True, exist_ok=True)
|
||||
(vdir / f"{version_id}.json").write_text(
|
||||
json.dumps(snapshot, ensure_ascii=False, indent=2, default=str),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
snapshot["_id"] = version_id
|
||||
return snapshot
|
||||
|
||||
|
||||
def diff_versions(project: str, v1: str, v2: str) -> dict[str, Any]:
|
||||
"""对比两个版本。"""
|
||||
a = get_version(project, v1)
|
||||
b = get_version(project, v2)
|
||||
if not a or not b:
|
||||
return {"error": "版本不存在"}
|
||||
|
||||
a_packs = set((a.get("packs") or {}).keys())
|
||||
b_packs = set((b.get("packs") or {}).keys())
|
||||
|
||||
pack_changes: list[dict[str, Any]] = []
|
||||
for pname in a_packs | b_packs:
|
||||
pa = (a.get("packs") or {}).get(pname, {})
|
||||
pb = (b.get("packs") or {}).get(pname, {})
|
||||
if pa != pb:
|
||||
pack_changes.append({
|
||||
"pack": pname,
|
||||
"v1": {"train": pa.get("train_images", 0), "val": pa.get("val_images", 0)},
|
||||
"v2": {"train": pb.get("train_images", 0), "val": pb.get("val_images", 0)},
|
||||
})
|
||||
|
||||
return {
|
||||
"v1": {"id": v1, "created": a.get("created_at"), "total": a.get("summary", {}).get("total_images", 0)},
|
||||
"v2": {"id": v2, "created": b.get("created_at"), "total": b.get("summary", {}).get("total_images", 0)},
|
||||
"added_packs": sorted(b_packs - a_packs),
|
||||
"removed_packs": sorted(a_packs - b_packs),
|
||||
"pack_changes": pack_changes,
|
||||
"image_delta": (b.get("summary", {}).get("total_images", 0) or 0) - (a.get("summary", {}).get("total_images", 0) or 0),
|
||||
}
|
||||
@@ -17,23 +17,31 @@ from as_platform.config import (
|
||||
MANIFESTS,
|
||||
)
|
||||
from as_platform.db.engine import Base, engine, session_scope
|
||||
from as_platform.db.models import Approval, Job, Permission, Role, User
|
||||
from as_platform.db.models import Approval, FleetVehicle, Job, OperationLog, Permission, Role, User
|
||||
|
||||
# 角色 → 权限
|
||||
ROLE_DEFS: dict[str, tuple[str, list[str]]] = {
|
||||
"admin": ("管理员", ["*"]),
|
||||
"reviewer": ("审核员", [
|
||||
"read:catalog", "read:pending", "read:jobs", "read:audit",
|
||||
"write:approval_review",
|
||||
"read:catalog", "read:pending", "read:jobs", "read:audit", "read:fleet", "read:deliveries",
|
||||
"write:approval_review", "write:approval_submit",
|
||||
]),
|
||||
"engineer": ("算法工程师", [
|
||||
"read:catalog", "read:pending", "read:jobs", "read:audit",
|
||||
"write:approval_submit",
|
||||
"read:catalog", "read:pending", "read:jobs", "read:audit", "read:fleet", "write:fleet",
|
||||
"write:approval_submit", "write:delivery_submit", "read:deliveries",
|
||||
"write:labeling_vendor", "write:labeling_assign",
|
||||
]),
|
||||
"labeler": ("标注协调", [
|
||||
"read:catalog", "read:pending", "write:approval_submit:register",
|
||||
"read:catalog", "read:pending", "read:fleet", "write:approval_submit:register",
|
||||
"write:delivery_submit", "read:deliveries", "write:labeling_assign",
|
||||
]),
|
||||
"viewer": ("只读访客", ["read:catalog", "read:pending"]),
|
||||
"internal_labeler": ("内部标注员", [
|
||||
"read:catalog", "read:pending", "read:jobs",
|
||||
]),
|
||||
"vendor_labeler": ("第三方标注", [
|
||||
"read:catalog", "read:pending", "read:jobs", "write:labeling_vendor",
|
||||
]),
|
||||
"viewer": ("只读访客", ["read:catalog", "read:pending", "read:fleet"]),
|
||||
}
|
||||
|
||||
PERMISSION_NAMES: dict[str, str] = {
|
||||
@@ -42,19 +50,37 @@ PERMISSION_NAMES: dict[str, str] = {
|
||||
"read:pending": "查看送标/批次",
|
||||
"read:jobs": "查看 Job 队列",
|
||||
"read:audit": "查看审核记录",
|
||||
"read:fleet": "查看车队地图",
|
||||
"write:fleet": "管理车队与轨迹",
|
||||
"write:approval_submit": "提交审核(训练/build 等)",
|
||||
"write:approval_submit:register": "提交批次登记审核",
|
||||
"write:approval_review": "批准/驳回审核",
|
||||
"admin:users": "用户与角色管理",
|
||||
"write:labeling_vendor": "导入第三方标注回传包",
|
||||
"write:labeling_assign": "分配标注子任务",
|
||||
"write:delivery_submit": "提交数据送标申请",
|
||||
"read:deliveries": "查看批次送标台账",
|
||||
}
|
||||
|
||||
|
||||
def init_database() -> None:
|
||||
MANIFESTS.mkdir(parents=True, exist_ok=True)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
# 标注质检表(audit/review.py,不通过 Base 自动发现)
|
||||
try:
|
||||
from as_platform.audit.review import LabelingReview
|
||||
LabelingReview.__table__.create(bind=engine, checkfirst=True)
|
||||
except Exception:
|
||||
pass
|
||||
with session_scope() as db:
|
||||
_ensure_user_columns(db)
|
||||
_ensure_dataset_candidate_columns(db)
|
||||
_ensure_labeling_campaign_columns(db)
|
||||
_ensure_feishu_bitable_columns(db)
|
||||
_ensure_approval_columns(db)
|
||||
_ensure_operation_log_columns(db)
|
||||
_seed_roles_permissions(db)
|
||||
_seed_fleet_demo(db)
|
||||
_import_jsonl_if_empty(db)
|
||||
_sync_postgres_sequences(db)
|
||||
|
||||
@@ -78,6 +104,20 @@ def _seed_roles_permissions(db: Session) -> None:
|
||||
role.permissions = [perm_map[c] for c in perm_codes if c in perm_map]
|
||||
|
||||
|
||||
|
||||
|
||||
def _seed_fleet_demo(db: Session) -> None:
|
||||
if db.query(FleetVehicle).count() > 0:
|
||||
return
|
||||
try:
|
||||
from as_platform.config import FLEET_MOCK_SEED
|
||||
if not FLEET_MOCK_SEED:
|
||||
return
|
||||
from as_platform.fleet.mock_seed import seed_demo_fleet
|
||||
seed_demo_fleet(db)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _import_jsonl_if_empty(db: Session) -> None:
|
||||
if db.query(Approval).count() == 0 and APPROVAL_QUEUE.is_file():
|
||||
for line in APPROVAL_QUEUE.read_text(encoding="utf-8").strip().splitlines():
|
||||
@@ -153,6 +193,8 @@ def user_has_permission(user: User, permission: str) -> bool:
|
||||
# register_batch 专用权限
|
||||
if permission == "write:approval_submit:register" and p.code == "write:approval_submit":
|
||||
return True
|
||||
if permission == "write:delivery_submit" and p.code == "write:approval_submit:register":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -222,6 +264,42 @@ def _sync_postgres_sequences(db: Session) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _ensure_dataset_candidate_columns(db: Session) -> None:
|
||||
columns = {
|
||||
"mode": "VARCHAR(64)",
|
||||
"inbox_path": "VARCHAR(1024)",
|
||||
"promoted_batch": "VARCHAR(128)",
|
||||
"external_id": "VARCHAR(128)",
|
||||
"feishu_record_id": "VARCHAR(64)",
|
||||
}
|
||||
_ensure_table_columns(db, "dataset_candidates", columns)
|
||||
|
||||
|
||||
def _ensure_feishu_bitable_columns(db: Session) -> None:
|
||||
_ensure_dataset_candidate_columns(db)
|
||||
|
||||
|
||||
def _ensure_labeling_campaign_columns(db: Session) -> None:
|
||||
columns = {
|
||||
"assigned_to_user_id": "INTEGER",
|
||||
"assigned_to_name": "VARCHAR(128)",
|
||||
}
|
||||
_ensure_table_columns(db, "labeling_campaigns", columns)
|
||||
|
||||
|
||||
def _ensure_table_columns(db: Session, table: str, columns: dict[str, str]) -> None:
|
||||
if IS_POSTGRES:
|
||||
for column, column_type in columns.items():
|
||||
db.execute(text(f"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS {column} {column_type}"))
|
||||
return
|
||||
if IS_SQLITE:
|
||||
rows = db.execute(text(f"PRAGMA table_info({table})")).fetchall()
|
||||
existing = {str(row[1]) for row in rows}
|
||||
for column, column_type in columns.items():
|
||||
if column not in existing:
|
||||
db.execute(text(f"ALTER TABLE {table} ADD COLUMN {column} {column_type}"))
|
||||
|
||||
|
||||
def _ensure_user_columns(db: Session) -> None:
|
||||
user_columns = {
|
||||
"feishu_user_id": "VARCHAR(64)",
|
||||
@@ -240,3 +318,20 @@ def _ensure_user_columns(db: Session) -> None:
|
||||
for column, column_type in user_columns.items():
|
||||
if column not in existing:
|
||||
db.execute(text(f"ALTER TABLE users ADD COLUMN {column} {column_type}"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _ensure_approval_columns(db: Session) -> None:
|
||||
_ensure_table_columns(db, "approvals", {"rejection_category": "VARCHAR(32) DEFAULT ''"})
|
||||
|
||||
|
||||
def _ensure_operation_log_columns(db: Session) -> None:
|
||||
"""确保 operation_logs 表存在并包含所有列。"""
|
||||
inspector_args = {}
|
||||
bind = db.get_bind()
|
||||
if hasattr(bind, "url") and "postgresql" in str(getattr(bind, "url", "")):
|
||||
inspector_args["schema"] = "public"
|
||||
try:
|
||||
OperationLog.__table__.create(bind=db.get_bind(), checkfirst=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -8,11 +8,13 @@ from sqlalchemy import (
|
||||
Boolean,
|
||||
Column,
|
||||
DateTime,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Table,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
@@ -104,6 +106,7 @@ class Approval(Base):
|
||||
reviewed_by_name = Column(String(128), nullable=True)
|
||||
reviewed_at = Column(DateTime(timezone=True), nullable=True)
|
||||
review_comment = Column(Text, nullable=True)
|
||||
rejection_category = Column(String(32), nullable=True, default="")
|
||||
job_id = Column(String(64), nullable=True)
|
||||
executed_at = Column(DateTime(timezone=True), nullable=True)
|
||||
result_json = Column(Text, nullable=True)
|
||||
@@ -137,6 +140,7 @@ class Approval(Base):
|
||||
"reviewed_by_user_id": self.reviewed_by_user_id,
|
||||
"reviewed_at": self.reviewed_at.isoformat() if self.reviewed_at else None,
|
||||
"review_comment": self.review_comment,
|
||||
"rejection_category": self.rejection_category or "",
|
||||
"job_id": self.job_id,
|
||||
"executed_at": self.executed_at.isoformat() if self.executed_at else None,
|
||||
"result": self.result(),
|
||||
@@ -190,6 +194,7 @@ class DatasetCandidate(Base):
|
||||
id = Column(String(64), primary_key=True)
|
||||
project = Column(String(32), nullable=False, index=True)
|
||||
task = Column(String(64), nullable=True, index=True)
|
||||
mode = Column(String(64), nullable=True)
|
||||
status = Column(String(32), nullable=False, default="uploaded", index=True)
|
||||
source_type = Column(String(32), nullable=False, default="upload")
|
||||
original_name = Column(String(255), nullable=True)
|
||||
@@ -203,6 +208,10 @@ class DatasetCandidate(Base):
|
||||
submitted_by_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
|
||||
submitted_by_name = Column(String(128), nullable=True)
|
||||
analysis_job_id = Column(String(64), ForeignKey("jobs.id"), nullable=True)
|
||||
inbox_path = Column(String(1024), nullable=True)
|
||||
promoted_batch = Column(String(128), nullable=True)
|
||||
external_id = Column(String(128), nullable=True, index=True)
|
||||
feishu_record_id = Column(String(64), nullable=True, index=True)
|
||||
created_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
@@ -225,6 +234,7 @@ class DatasetCandidate(Base):
|
||||
"id": self.id,
|
||||
"project": self.project,
|
||||
"task": self.task,
|
||||
"mode": self.mode,
|
||||
"status": self.status,
|
||||
"source_type": self.source_type,
|
||||
"original_name": self.original_name,
|
||||
@@ -238,6 +248,369 @@ class DatasetCandidate(Base):
|
||||
"submitted_by_user_id": self.submitted_by_user_id,
|
||||
"submitted_by_name": self.submitted_by_name,
|
||||
"analysis_job_id": self.analysis_job_id,
|
||||
"inbox_path": self.inbox_path,
|
||||
"promoted_batch": self.promoted_batch,
|
||||
"external_id": self.external_id,
|
||||
"feishu_record_id": self.feishu_record_id,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
class BatchDelivery(Base):
|
||||
"""平台内数据送标申请(替代飞书多维表格拉取)。"""
|
||||
|
||||
__tablename__ = "batch_deliveries"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("project", "task", "mode", "batch_name", name="uq_batch_deliveries_batch"),
|
||||
)
|
||||
|
||||
id = Column(String(64), primary_key=True)
|
||||
project = Column(String(32), nullable=False, index=True)
|
||||
task = Column(String(64), nullable=True, index=True)
|
||||
mode = Column(String(64), nullable=True)
|
||||
batch_name = Column(String(128), nullable=False, index=True)
|
||||
source_type = Column(String(64), nullable=True)
|
||||
vehicle_scene = Column(String(256), nullable=True)
|
||||
collection_start = Column(String(32), nullable=True)
|
||||
collection_end = Column(String(32), nullable=True)
|
||||
data_path = Column(String(1024), nullable=False)
|
||||
estimated_count = Column(Integer, nullable=True)
|
||||
remark = Column(Text, nullable=True)
|
||||
status = Column(String(32), nullable=False, default="draft", index=True)
|
||||
owner_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
|
||||
owner_name = Column(String(128), nullable=True)
|
||||
submitted_by_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
|
||||
submitted_by_name = Column(String(128), nullable=True)
|
||||
approval_id = Column(String(64), nullable=True, index=True)
|
||||
candidate_id = Column(String(64), nullable=True, index=True)
|
||||
inbox_path = Column(String(1024), nullable=True)
|
||||
error_message = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"project": self.project,
|
||||
"task": self.task,
|
||||
"mode": self.mode,
|
||||
"batch_name": self.batch_name,
|
||||
"source_type": self.source_type,
|
||||
"vehicle_scene": self.vehicle_scene,
|
||||
"collection_start": self.collection_start,
|
||||
"collection_end": self.collection_end,
|
||||
"data_path": self.data_path,
|
||||
"estimated_count": self.estimated_count,
|
||||
"remark": self.remark,
|
||||
"status": self.status,
|
||||
"owner_user_id": self.owner_user_id,
|
||||
"owner_name": self.owner_name,
|
||||
"submitted_by_user_id": self.submitted_by_user_id,
|
||||
"submitted_by_name": self.submitted_by_name,
|
||||
"submitted_by": self.submitted_by_name,
|
||||
"approval_id": self.approval_id,
|
||||
"candidate_id": self.candidate_id,
|
||||
"inbox_path": self.inbox_path,
|
||||
"error_message": self.error_message,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
class FeishuBitableLink(Base):
|
||||
"""HSAP 批次与飞书多维表格行的对应关系。"""
|
||||
|
||||
__tablename__ = "feishu_bitable_links"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
batch_key = Column(String(256), unique=True, nullable=False, index=True)
|
||||
record_id = Column(String(64), unique=True, nullable=True, index=True)
|
||||
delivery_id = Column(String(128), nullable=True, index=True)
|
||||
project = Column(String(32), nullable=False)
|
||||
task = Column(String(64), nullable=True)
|
||||
mode = Column(String(64), nullable=True)
|
||||
batch = Column(String(128), nullable=False)
|
||||
campaign_id = Column(String(64), nullable=True)
|
||||
inbox_path = Column(String(1024), nullable=True)
|
||||
last_sync_at = Column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
|
||||
class FleetVehicle(Base):
|
||||
__tablename__ = "fleet_vehicles"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
plate_no = Column(String(32), nullable=False, index=True)
|
||||
tbox_device_id = Column(String(64), unique=True, nullable=False, index=True)
|
||||
name = Column(String(128), nullable=False, default="")
|
||||
team = Column(String(64), nullable=True)
|
||||
status = Column(String(32), nullable=False, default="active")
|
||||
last_lat = Column(Float, nullable=True)
|
||||
last_lng = Column(Float, nullable=True)
|
||||
last_speed_kmh = Column(Float, nullable=True)
|
||||
last_ts = Column(DateTime(timezone=True), nullable=True)
|
||||
online = Column(Boolean, nullable=False, default=False)
|
||||
meta_json = Column(Text, nullable=False, default="{}")
|
||||
created_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
def meta(self) -> dict:
|
||||
return json.loads(self.meta_json or "{}")
|
||||
|
||||
def set_meta(self, data: dict) -> None:
|
||||
self.meta_json = json.dumps(data, ensure_ascii=False)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"plate_no": self.plate_no,
|
||||
"tbox_device_id": self.tbox_device_id,
|
||||
"name": self.name or self.plate_no,
|
||||
"team": self.team,
|
||||
"status": self.status,
|
||||
"last_lat": self.last_lat,
|
||||
"last_lng": self.last_lng,
|
||||
"last_speed_kmh": self.last_speed_kmh,
|
||||
"last_ts": self.last_ts.isoformat() if self.last_ts else None,
|
||||
"online": self.online,
|
||||
"meta": self.meta(),
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
class FleetCollectionRun(Base):
|
||||
__tablename__ = "fleet_collection_runs"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
vehicle_id = Column(Integer, ForeignKey("fleet_vehicles.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
run_no = Column(String(64), nullable=False, index=True)
|
||||
engineer = Column(String(128), nullable=True)
|
||||
project = Column(String(32), nullable=True)
|
||||
batch = Column(String(128), nullable=True)
|
||||
started_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
ended_at = Column(DateTime(timezone=True), nullable=True)
|
||||
status = Column(String(32), nullable=False, default="active", index=True)
|
||||
mileage_km = Column(Float, nullable=False, default=0.0)
|
||||
source = Column(String(32), nullable=False, default="tbox")
|
||||
note = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"vehicle_id": self.vehicle_id,
|
||||
"run_no": self.run_no,
|
||||
"engineer": self.engineer,
|
||||
"project": self.project,
|
||||
"batch": self.batch,
|
||||
"started_at": self.started_at.isoformat() if self.started_at else None,
|
||||
"ended_at": self.ended_at.isoformat() if self.ended_at else None,
|
||||
"status": self.status,
|
||||
"mileage_km": round(self.mileage_km or 0.0, 3),
|
||||
"source": self.source,
|
||||
"note": self.note,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
class FleetTrackPoint(Base):
|
||||
__tablename__ = "fleet_track_points"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
run_id = Column(Integer, ForeignKey("fleet_collection_runs.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
ts = Column(DateTime(timezone=True), nullable=False, index=True)
|
||||
lat = Column(Float, nullable=False)
|
||||
lng = Column(Float, nullable=False)
|
||||
speed_kmh = Column(Float, nullable=True)
|
||||
heading = Column(Float, nullable=True)
|
||||
alt_m = Column(Float, nullable=True)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"run_id": self.run_id,
|
||||
"ts": self.ts.isoformat() if self.ts else None,
|
||||
"lat": self.lat,
|
||||
"lng": self.lng,
|
||||
"speed_kmh": self.speed_kmh,
|
||||
"heading": self.heading,
|
||||
"alt_m": self.alt_m,
|
||||
}
|
||||
|
||||
|
||||
class FleetRunMilestone(Base):
|
||||
__tablename__ = "fleet_run_milestones"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
run_id = Column(Integer, ForeignKey("fleet_collection_runs.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
name = Column(String(128), nullable=False, default="")
|
||||
type = Column(String(32), nullable=False, default="waypoint")
|
||||
lat = Column(Float, nullable=False)
|
||||
lng = Column(Float, nullable=False)
|
||||
mileage_km = Column(Float, nullable=True)
|
||||
occurred_at = Column(DateTime(timezone=True), nullable=True)
|
||||
payload_json = Column(Text, nullable=False, default="{}")
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"run_id": self.run_id,
|
||||
"name": self.name,
|
||||
"type": self.type,
|
||||
"lat": self.lat,
|
||||
"lng": self.lng,
|
||||
"mileage_km": self.mileage_km,
|
||||
"occurred_at": self.occurred_at.isoformat() if self.occurred_at else None,
|
||||
"payload": json.loads(self.payload_json or "{}"),
|
||||
}
|
||||
|
||||
|
||||
class LabelingCampaignAccess(Base):
|
||||
"""Campaign 级访问授权(内部/第三方),非派单。"""
|
||||
|
||||
__tablename__ = "labeling_campaign_access"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
campaign_id = Column(String(64), nullable=False, index=True)
|
||||
principal_type = Column(String(32), nullable=False, default="role")
|
||||
principal_id = Column(String(128), nullable=False, index=True)
|
||||
access_role = Column(String(32), nullable=False, default="vendor")
|
||||
created_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"campaign_id": self.campaign_id,
|
||||
"principal_type": self.principal_type,
|
||||
"principal_id": self.principal_id,
|
||||
"access_role": self.access_role,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
class LabelingExportJob(Base):
|
||||
"""标注活动关联的 export / ml-predict 任务记录。"""
|
||||
|
||||
__tablename__ = "labeling_export_jobs"
|
||||
|
||||
id = Column(String(64), primary_key=True)
|
||||
campaign_id = Column(String(64), nullable=False, index=True)
|
||||
action = Column(String(64), nullable=False, index=True)
|
||||
job_id = Column(String(64), nullable=True, index=True)
|
||||
status = Column(String(32), nullable=False, default="queued", index=True)
|
||||
result_json = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
finished_at = Column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"campaign_id": self.campaign_id,
|
||||
"action": self.action,
|
||||
"job_id": self.job_id,
|
||||
"status": self.status,
|
||||
"result": json.loads(self.result_json or "null") if self.result_json else None,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"finished_at": self.finished_at.isoformat() if self.finished_at else None,
|
||||
}
|
||||
|
||||
|
||||
class LabelingCampaign(Base):
|
||||
"""标注活动:与 pending 批次 (project, task, mode, batch, location) 一一对应。"""
|
||||
|
||||
__tablename__ = "labeling_campaigns"
|
||||
|
||||
id = Column(String(64), primary_key=True)
|
||||
project = Column(String(32), nullable=False, index=True)
|
||||
task = Column(String(64), nullable=False, index=True)
|
||||
mode = Column(String(64), nullable=True, index=True)
|
||||
batch = Column(String(128), nullable=False, index=True)
|
||||
pack = Column(String(64), nullable=True)
|
||||
location = Column(String(32), nullable=False, default="inbox")
|
||||
status = Column(String(32), nullable=False, default="not_opened", index=True)
|
||||
assigned_to_user_id = Column(Integer, ForeignKey("users.id"), nullable=True, index=True)
|
||||
assigned_to_name = Column(String(128), nullable=True)
|
||||
config_xml = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
updated_at = Column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"project": self.project,
|
||||
"task": self.task,
|
||||
"mode": self.mode,
|
||||
"batch": self.batch,
|
||||
"pack": self.pack,
|
||||
"location": self.location,
|
||||
"status": self.status,
|
||||
"assigned_to_user_id": self.assigned_to_user_id,
|
||||
"assigned_to_name": self.assigned_to_name,
|
||||
"config_xml": self.config_xml,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
class LabelingTaskAssignment(Base):
|
||||
"""Campaign 内单张图(task_id)分包给标注员。"""
|
||||
|
||||
__tablename__ = "labeling_task_assignments"
|
||||
__table_args__ = (UniqueConstraint("campaign_id", "task_id", name="uq_labeling_campaign_task"),)
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
campaign_id = Column(String(64), ForeignKey("labeling_campaigns.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
task_id = Column(String(32), nullable=False, index=True)
|
||||
user_id = Column(Integer, ForeignKey("users.id"), nullable=False, index=True)
|
||||
assigned_by_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
|
||||
assigned_at = Column(DateTime(timezone=True), default=_utcnow)
|
||||
completed_at = Column(DateTime(timezone=True), nullable=True)
|
||||
completed_by_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"campaign_id": self.campaign_id,
|
||||
"task_id": self.task_id,
|
||||
"user_id": self.user_id,
|
||||
"assigned_by_user_id": self.assigned_by_user_id,
|
||||
"assigned_at": self.assigned_at.isoformat() if self.assigned_at else None,
|
||||
"completed_at": self.completed_at.isoformat() if self.completed_at else None,
|
||||
"completed_by_user_id": self.completed_by_user_id,
|
||||
}
|
||||
|
||||
|
||||
class OperationLog(Base):
|
||||
__tablename__ = "operation_logs"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
timestamp = Column(DateTime(timezone=True), default=_utcnow, index=True)
|
||||
user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
|
||||
user_name = Column(String(128), nullable=True)
|
||||
category = Column(String(32), nullable=False, index=True)
|
||||
action = Column(String(64), nullable=False)
|
||||
target_type = Column(String(32), nullable=True)
|
||||
target_id = Column(String(128), nullable=True)
|
||||
summary = Column(String(512), nullable=True)
|
||||
detail_json = Column(Text, nullable=True)
|
||||
ip_address = Column(String(64), nullable=True)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"timestamp": self.timestamp.isoformat() if self.timestamp else None,
|
||||
"user_id": self.user_id,
|
||||
"user_name": self.user_name,
|
||||
"category": self.category,
|
||||
"action": self.action,
|
||||
"target_type": self.target_type,
|
||||
"target_id": self.target_id,
|
||||
"summary": self.summary,
|
||||
"detail": json.loads(self.detail_json) if self.detail_json else None,
|
||||
"ip_address": self.ip_address,
|
||||
}
|
||||
|
||||
|
||||
|
||||
1
platform/as_platform/deliveries/__init__.py
Normal file
1
platform/as_platform/deliveries/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""平台批次送标申请。"""
|
||||
314
platform/as_platform/deliveries/service.py
Normal file
314
platform/as_platform/deliveries/service.py
Normal file
@@ -0,0 +1,314 @@
|
||||
"""批次送标申请 CRUD 与提交审批。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from as_platform.audit.queue import reject_approval, submit_approval
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import Approval, BatchDelivery, Job, User
|
||||
from as_platform.integrations.delivery_ingest import validate_delivery_fields
|
||||
|
||||
EDITABLE_STATUSES = frozenset({"draft", "rejected", "ingest_failed"})
|
||||
DELETABLE_STATUSES = frozenset({"draft", "rejected", "ingest_failed"})
|
||||
RETRY_INGEST_STATUSES = frozenset({"ingest_failed", "ingesting"})
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _new_delivery_id() -> str:
|
||||
return f"del-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def _normalize_task(project: str, task: str | None) -> str | None:
|
||||
if project == "dms":
|
||||
return (task or "").strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def _enrich_delivery_dict(db, rec: BatchDelivery) -> dict[str, Any]:
|
||||
d = rec.to_dict()
|
||||
d["submitted_by"] = rec.submitted_by_name
|
||||
if rec.approval_id:
|
||||
ap = db.get(Approval, rec.approval_id)
|
||||
if ap:
|
||||
d["approval_status"] = ap.status
|
||||
d["job_id"] = ap.job_id
|
||||
if ap.job_id:
|
||||
job = db.get(Job, ap.job_id)
|
||||
if job:
|
||||
d["job_status"] = job.status
|
||||
return d
|
||||
|
||||
|
||||
def list_deliveries(
|
||||
*,
|
||||
status: str | None = None,
|
||||
mine_user_id: int | None = None,
|
||||
mine_editable_only: bool = False,
|
||||
offset: int = 0,
|
||||
limit: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
q = db.query(BatchDelivery).order_by(BatchDelivery.updated_at.desc())
|
||||
if status:
|
||||
q = q.filter(BatchDelivery.status == status)
|
||||
if mine_user_id is not None:
|
||||
q = q.filter(
|
||||
(BatchDelivery.owner_user_id == mine_user_id)
|
||||
| (BatchDelivery.submitted_by_user_id == mine_user_id)
|
||||
)
|
||||
if mine_editable_only:
|
||||
q = q.filter(BatchDelivery.status.in_(("draft", "rejected", "ingest_failed")))
|
||||
total = q.count()
|
||||
rows = q.offset(max(0, offset)).limit(max(1, limit)).all()
|
||||
return {
|
||||
"items": [_enrich_delivery_dict(db, r) for r in rows],
|
||||
"total": total,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"count": len(rows),
|
||||
}
|
||||
|
||||
|
||||
def get_delivery(delivery_id: str) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchDelivery, delivery_id)
|
||||
return _enrich_delivery_dict(db, rec) if rec else None
|
||||
|
||||
|
||||
def create_delivery(data: dict[str, Any], user: User) -> dict[str, Any]:
|
||||
project = (data.get("project") or "dms").strip()
|
||||
task = _normalize_task(project, data.get("task"))
|
||||
mode = (data.get("mode") or "").strip() or None
|
||||
batch_name = (data.get("batch_name") or "").strip()
|
||||
data_path = (data.get("data_path") or "").strip()
|
||||
|
||||
if not batch_name:
|
||||
raise ValueError("批次名必填")
|
||||
if data_path:
|
||||
err = validate_delivery_fields(
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
batch_name=batch_name,
|
||||
data_path=data_path,
|
||||
)
|
||||
if err:
|
||||
raise ValueError(err)
|
||||
|
||||
with session_scope() as db:
|
||||
dup = (
|
||||
db.query(BatchDelivery)
|
||||
.filter_by(project=project, task=task, mode=mode, batch_name=batch_name)
|
||||
.filter(BatchDelivery.status.notin_(("rejected", "ingest_failed")))
|
||||
.first()
|
||||
)
|
||||
if dup:
|
||||
raise ValueError(f"已存在同批次申请: {dup.id} ({dup.status})")
|
||||
|
||||
rec = BatchDelivery(
|
||||
id=_new_delivery_id(),
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
batch_name=batch_name,
|
||||
source_type=(data.get("source_type") or "").strip() or None,
|
||||
vehicle_scene=(data.get("vehicle_scene") or "").strip() or None,
|
||||
collection_start=(data.get("collection_start") or "").strip() or None,
|
||||
collection_end=(data.get("collection_end") or "").strip() or None,
|
||||
data_path=data_path,
|
||||
estimated_count=int(data["estimated_count"]) if data.get("estimated_count") not in (None, "") else None,
|
||||
remark=(data.get("remark") or "").strip() or None,
|
||||
status="draft",
|
||||
owner_user_id=data.get("owner_user_id") or user.id,
|
||||
owner_name=(data.get("owner_name") or user.name),
|
||||
submitted_by_user_id=user.id,
|
||||
submitted_by_name=user.name,
|
||||
)
|
||||
db.add(rec)
|
||||
db.flush()
|
||||
return rec.to_dict()
|
||||
|
||||
|
||||
def update_delivery(delivery_id: str, data: dict[str, Any], user: User) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchDelivery, delivery_id)
|
||||
if not rec:
|
||||
raise ValueError("送标申请不存在")
|
||||
if rec.status not in EDITABLE_STATUSES:
|
||||
raise ValueError(f"当前状态不可编辑: {rec.status}")
|
||||
|
||||
project = (data.get("project") or rec.project).strip()
|
||||
task = _normalize_task(project, data.get("task") if "task" in data else rec.task)
|
||||
mode = (data.get("mode") if "mode" in data else rec.mode) or None
|
||||
if mode:
|
||||
mode = mode.strip() or None
|
||||
batch_name = (data.get("batch_name") or rec.batch_name).strip()
|
||||
data_path = (data.get("data_path") or rec.data_path).strip()
|
||||
|
||||
err = validate_delivery_fields(
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
batch_name=batch_name,
|
||||
data_path=data_path,
|
||||
)
|
||||
if err:
|
||||
raise ValueError(err)
|
||||
|
||||
rec.project = project
|
||||
rec.task = task
|
||||
rec.mode = mode
|
||||
rec.batch_name = batch_name
|
||||
rec.data_path = data_path
|
||||
if "source_type" in data:
|
||||
rec.source_type = (data.get("source_type") or "").strip() or None
|
||||
if "vehicle_scene" in data:
|
||||
rec.vehicle_scene = (data.get("vehicle_scene") or "").strip() or None
|
||||
if "collection_start" in data:
|
||||
rec.collection_start = (data.get("collection_start") or "").strip() or None
|
||||
if "collection_end" in data:
|
||||
rec.collection_end = (data.get("collection_end") or "").strip() or None
|
||||
if "estimated_count" in data:
|
||||
v = data.get("estimated_count")
|
||||
rec.estimated_count = int(v) if v not in (None, "") else None
|
||||
if "remark" in data:
|
||||
rec.remark = (data.get("remark") or "").strip() or None
|
||||
if "owner_user_id" in data:
|
||||
rec.owner_user_id = data.get("owner_user_id") or rec.owner_user_id
|
||||
if "owner_name" in data:
|
||||
rec.owner_name = (data.get("owner_name") or "").strip() or rec.owner_name
|
||||
if rec.status in ("rejected", "ingest_failed"):
|
||||
rec.status = "draft"
|
||||
rec.error_message = None
|
||||
rec.updated_at = _utcnow()
|
||||
db.flush()
|
||||
return rec.to_dict()
|
||||
|
||||
|
||||
def submit_delivery_for_review(delivery_id: str, user: User) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchDelivery, delivery_id)
|
||||
if not rec:
|
||||
raise ValueError("送标申请不存在")
|
||||
if rec.status not in EDITABLE_STATUSES:
|
||||
raise ValueError(f"当前状态不可提交: {rec.status}")
|
||||
|
||||
err = validate_delivery_fields(
|
||||
project=rec.project,
|
||||
task=rec.task,
|
||||
mode=rec.mode,
|
||||
batch_name=rec.batch_name,
|
||||
data_path=rec.data_path,
|
||||
)
|
||||
if err:
|
||||
raise ValueError(err)
|
||||
|
||||
params = {
|
||||
"delivery_id": rec.id,
|
||||
"project": rec.project,
|
||||
"task": rec.task,
|
||||
"mode": rec.mode,
|
||||
"batch_name": rec.batch_name,
|
||||
"data_path": rec.data_path,
|
||||
"source_type": rec.source_type,
|
||||
"vehicle_scene": rec.vehicle_scene,
|
||||
"estimated_count": rec.estimated_count,
|
||||
"remark": rec.remark,
|
||||
}
|
||||
approval = submit_approval(
|
||||
"delivery_ingest",
|
||||
params,
|
||||
submitted_by=user.name,
|
||||
submitted_by_user_id=user.id,
|
||||
note=f"数据送标入湖 {rec.batch_name}",
|
||||
)
|
||||
rec.status = "pending_review"
|
||||
rec.approval_id = approval.get("id")
|
||||
rec.submitted_by_user_id = user.id
|
||||
rec.submitted_by_name = user.name
|
||||
rec.updated_at = _utcnow()
|
||||
db.flush()
|
||||
out = rec.to_dict()
|
||||
out["approval"] = approval
|
||||
return out
|
||||
|
||||
|
||||
def mark_delivery_rejected_by_approval(approval_id: str) -> None:
|
||||
with session_scope() as db:
|
||||
rec = db.query(BatchDelivery).filter_by(approval_id=approval_id).first()
|
||||
if rec and rec.status == "pending_review":
|
||||
rec.status = "rejected"
|
||||
rec.updated_at = _utcnow()
|
||||
db.flush()
|
||||
|
||||
|
||||
def on_approval_rejected(approval_id: str, **_: Any) -> None:
|
||||
mark_delivery_rejected_by_approval(approval_id)
|
||||
|
||||
|
||||
def mark_delivery_ingest_failed(delivery_id: str | None, approval_id: str | None, error: str) -> None:
|
||||
"""Job 失败或入湖异常时,确保台账状态为 ingest_failed(避免卡在 ingesting)。"""
|
||||
msg = (error or "")[:2000]
|
||||
with session_scope() as db:
|
||||
rec = None
|
||||
if delivery_id:
|
||||
rec = db.get(BatchDelivery, delivery_id)
|
||||
if not rec and approval_id:
|
||||
rec = db.query(BatchDelivery).filter_by(approval_id=approval_id).first()
|
||||
if not rec:
|
||||
return
|
||||
if rec.status in ("ingesting", "pending_review"):
|
||||
rec.status = "ingest_failed"
|
||||
rec.error_message = msg
|
||||
rec.updated_at = _utcnow()
|
||||
db.flush()
|
||||
|
||||
|
||||
def delete_delivery(delivery_id: str, user: User) -> None:
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchDelivery, delivery_id)
|
||||
if not rec:
|
||||
raise ValueError("送标申请不存在")
|
||||
if rec.status not in DELETABLE_STATUSES:
|
||||
raise ValueError(f"当前状态不可删除: {rec.status}")
|
||||
db.delete(rec)
|
||||
db.flush()
|
||||
|
||||
|
||||
def retry_delivery_ingest(delivery_id: str, user: User) -> dict[str, Any]:
|
||||
"""入湖失败后重新执行入湖 Job(无需重新走审核)。"""
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchDelivery, delivery_id)
|
||||
if not rec:
|
||||
raise ValueError("送标申请不存在")
|
||||
if rec.status not in RETRY_INGEST_STATUSES:
|
||||
raise ValueError(f"当前状态不可重新入湖: {rec.status}")
|
||||
err = validate_delivery_fields(
|
||||
project=rec.project,
|
||||
task=rec.task,
|
||||
mode=rec.mode,
|
||||
batch_name=rec.batch_name,
|
||||
data_path=rec.data_path,
|
||||
)
|
||||
if err:
|
||||
raise ValueError(err)
|
||||
approval_id = rec.approval_id
|
||||
rec.status = "ingesting"
|
||||
rec.error_message = None
|
||||
rec.updated_at = _utcnow()
|
||||
db.flush()
|
||||
|
||||
from as_platform.jobs.queue import enqueue_job
|
||||
|
||||
job = enqueue_job(
|
||||
"delivery_ingest",
|
||||
{"delivery_id": delivery_id},
|
||||
approval_id=approval_id,
|
||||
async_run=True,
|
||||
)
|
||||
return {"ok": True, "job_id": job.get("id"), "delivery_id": delivery_id}
|
||||
1
platform/as_platform/fleet/.write_test
Normal file
1
platform/as_platform/fleet/.write_test
Normal file
@@ -0,0 +1 @@
|
||||
ok
|
||||
0
platform/as_platform/fleet/__init__.py
Normal file
0
platform/as_platform/fleet/__init__.py
Normal file
21
platform/as_platform/fleet/geo.py
Normal file
21
platform/as_platform/fleet/geo.py
Normal file
@@ -0,0 +1,21 @@
|
||||
"""Fleet geo helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
EARTH_RADIUS_KM = 6371.0
|
||||
|
||||
|
||||
def haversine_km(lat1: float, lng1: float, lat2: float, lng2: float) -> float:
|
||||
rlat1, rlng1, rlat2, rlng2 = map(math.radians, (lat1, lng1, lat2, lng2))
|
||||
dlat = rlat2 - rlat1
|
||||
dlng = rlng2 - rlng1
|
||||
a = math.sin(dlat / 2) ** 2 + math.cos(rlat1) * math.cos(rlat2) * math.sin(dlng / 2) ** 2
|
||||
return 2 * EARTH_RADIUS_KM * math.asin(math.sqrt(a))
|
||||
|
||||
|
||||
def simplify_coords(coords: list[list[float]], max_points: int = 2000) -> list[list[float]]:
|
||||
if len(coords) <= max_points:
|
||||
return coords
|
||||
step = max(1, len(coords) // max_points)
|
||||
return [coords[i] for i in range(0, len(coords), step)]
|
||||
212
platform/as_platform/fleet/mock_seed.py
Normal file
212
platform/as_platform/fleet/mock_seed.py
Normal file
@@ -0,0 +1,212 @@
|
||||
"""Demo fleet seed data (Changsha area, curved paths)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from as_platform.db.models import (
|
||||
FleetCollectionRun,
|
||||
FleetRunMilestone,
|
||||
FleetTrackPoint,
|
||||
FleetVehicle,
|
||||
)
|
||||
from as_platform.fleet.geo import haversine_km
|
||||
|
||||
|
||||
def _catmull_rom_point(
|
||||
p0: tuple[float, float],
|
||||
p1: tuple[float, float],
|
||||
p2: tuple[float, float],
|
||||
p3: tuple[float, float],
|
||||
t: float,
|
||||
) -> tuple[float, float]:
|
||||
t2, t3 = t * t, t * t * t
|
||||
lat = 0.5 * (
|
||||
(2 * p1[0])
|
||||
+ (-p0[0] + p2[0]) * t
|
||||
+ (2 * p0[0] - 5 * p1[0] + 4 * p2[0] - p3[0]) * t2
|
||||
+ (-p0[0] + 3 * p1[0] - 3 * p2[0] + p3[0]) * t3
|
||||
)
|
||||
lng = 0.5 * (
|
||||
(2 * p1[1])
|
||||
+ (-p0[1] + p2[1]) * t
|
||||
+ (2 * p0[1] - 5 * p1[1] + 4 * p2[1] - p3[1]) * t2
|
||||
+ (-p0[1] + 3 * p1[1] - 3 * p2[1] + p3[1]) * t3
|
||||
)
|
||||
return (lat, lng)
|
||||
|
||||
|
||||
def build_curved_route(anchors: list[tuple[float, float]], steps_per_seg: int = 24) -> list[tuple[float, float]]:
|
||||
"""Catmull-Rom 样条密点,地图折线呈弯道。"""
|
||||
if len(anchors) < 2:
|
||||
return list(anchors)
|
||||
pts = [anchors[0], *anchors, anchors[-1]]
|
||||
path: list[tuple[float, float]] = []
|
||||
for i in range(1, len(pts) - 2):
|
||||
p0, p1, p2, p3 = pts[i - 1], pts[i], pts[i + 1], pts[i + 2]
|
||||
for j in range(steps_per_seg):
|
||||
t = j / steps_per_seg
|
||||
path.append(_catmull_rom_point(p0, p1, p2, p3, t))
|
||||
path.append(anchors[-1])
|
||||
return path
|
||||
|
||||
|
||||
def _wiggle(path: list[tuple[float, float]], amp: float = 0.004) -> list[tuple[float, float]]:
|
||||
"""叠加轻微蛇形偏移,避免视觉上像直线。"""
|
||||
out: list[tuple[float, float]] = []
|
||||
for i, (lat, lng) in enumerate(path):
|
||||
w = math.sin(i * 0.35) * amp
|
||||
out.append((lat + w, lng - w * 0.6))
|
||||
return out
|
||||
|
||||
|
||||
# 长沙:锚点刻意折线(之字形 / 弧形),再样条平滑
|
||||
_ANCHORS: dict[str, list[tuple[float, float]]] = {
|
||||
"TBOX-001": [
|
||||
(28.172, 112.902),
|
||||
(28.188, 112.918),
|
||||
(28.205, 112.908),
|
||||
(28.222, 112.928),
|
||||
(28.238, 112.948),
|
||||
(28.252, 112.978),
|
||||
(28.248, 113.012),
|
||||
(28.235, 113.042),
|
||||
(28.218, 113.058),
|
||||
(28.202, 113.048),
|
||||
],
|
||||
"TBOX-002": [
|
||||
(28.278, 112.948),
|
||||
(28.270, 112.972),
|
||||
(28.255, 113.002),
|
||||
(28.238, 113.028),
|
||||
(28.218, 113.048),
|
||||
(28.202, 113.038),
|
||||
(28.198, 113.008),
|
||||
(28.208, 112.978),
|
||||
(28.225, 112.958),
|
||||
(28.248, 112.952),
|
||||
],
|
||||
"TBOX-003": [
|
||||
(28.212, 112.918),
|
||||
(28.225, 112.938),
|
||||
(28.242, 112.928),
|
||||
(28.258, 112.948),
|
||||
(28.272, 112.972),
|
||||
(28.268, 113.002),
|
||||
(28.252, 113.022),
|
||||
(28.232, 113.028),
|
||||
(28.215, 113.012),
|
||||
(28.208, 112.982),
|
||||
(28.212, 112.918),
|
||||
],
|
||||
}
|
||||
|
||||
ROUTES: dict[str, list[tuple[float, float]]] = {
|
||||
k: _wiggle(build_curved_route(v, steps_per_seg=28), amp=0.003)
|
||||
for k, v in _ANCHORS.items()
|
||||
}
|
||||
|
||||
VEHICLES = [
|
||||
("TBOX-001", "湘A·采集01", "岳麓采集车 A", "数据部"),
|
||||
("TBOX-002", "湘A·采集02", "开福采集车 B", "数据部"),
|
||||
("TBOX-003", "湘A·采集03", "天心采集车 C", "数据部"),
|
||||
]
|
||||
|
||||
|
||||
def _add_points(db: Session, run_id: int, coords: list[tuple[float, float]], start: datetime, interval_sec: int = 12) -> float:
|
||||
mileage = 0.0
|
||||
prev = None
|
||||
for i, (lat, lng) in enumerate(coords):
|
||||
ts = start + timedelta(seconds=i * interval_sec)
|
||||
speed = 35.0 + (i % 5) * 3.0
|
||||
pt = FleetTrackPoint(run_id=run_id, ts=ts, lat=lat, lng=lng, speed_kmh=speed, heading=float(i * 10 % 360))
|
||||
db.add(pt)
|
||||
if prev:
|
||||
mileage += haversine_km(prev[0], prev[1], lat, lng)
|
||||
prev = (lat, lng)
|
||||
return mileage
|
||||
|
||||
|
||||
def clear_demo_fleet(db: Session) -> None:
|
||||
db.query(FleetRunMilestone).delete()
|
||||
db.query(FleetTrackPoint).delete()
|
||||
db.query(FleetCollectionRun).delete()
|
||||
db.query(FleetVehicle).delete()
|
||||
db.flush()
|
||||
|
||||
|
||||
def seed_demo_fleet(db: Session) -> bool:
|
||||
if db.query(FleetVehicle).count() > 0:
|
||||
return False
|
||||
now = datetime.now(timezone.utc)
|
||||
for device_id, plate, name, team in VEHICLES:
|
||||
coords = ROUTES[device_id]
|
||||
v = FleetVehicle(
|
||||
plate_no=plate,
|
||||
tbox_device_id=device_id,
|
||||
name=name,
|
||||
team=team,
|
||||
status="active",
|
||||
online=True,
|
||||
last_lat=coords[-1][0],
|
||||
last_lng=coords[-1][1],
|
||||
last_speed_kmh=42.0,
|
||||
last_ts=now,
|
||||
)
|
||||
v.set_meta({"sim_index": 0, "sim_route": device_id, "sim_dir": 1})
|
||||
db.add(v)
|
||||
db.flush()
|
||||
|
||||
ended_start = now - timedelta(hours=6)
|
||||
ended = FleetCollectionRun(
|
||||
vehicle_id=v.id,
|
||||
run_no=f"{device_id}-{(ended_start.strftime('%Y%m%d'))}-01",
|
||||
engineer="陈工",
|
||||
project="dms",
|
||||
batch="demo_batch_01",
|
||||
started_at=ended_start,
|
||||
ended_at=ended_start + timedelta(hours=2),
|
||||
status="ended",
|
||||
source="mock",
|
||||
note="演示历史趟次(长沙弯道)",
|
||||
)
|
||||
db.add(ended)
|
||||
db.flush()
|
||||
em = _add_points(db, ended.id, coords, ended_start)
|
||||
ended.mileage_km = round(em, 3)
|
||||
db.add(FleetRunMilestone(run_id=ended.id, type="start", name="出发", lat=coords[0][0], lng=coords[0][1], mileage_km=0.0, occurred_at=ended_start))
|
||||
db.add(FleetRunMilestone(run_id=ended.id, type="end", name="结束", lat=coords[-1][0], lng=coords[-1][1], mileage_km=ended.mileage_km, occurred_at=ended.ended_at))
|
||||
db.add(FleetRunMilestone(run_id=ended.id, type="data_site", name="采集点-1", lat=coords[len(coords)//2][0], lng=coords[len(coords)//2][1], mileage_km=ended.mileage_km / 2, occurred_at=ended_start + timedelta(hours=1)))
|
||||
|
||||
active_start = now - timedelta(minutes=45)
|
||||
active = FleetCollectionRun(
|
||||
vehicle_id=v.id,
|
||||
run_no=f"{device_id}-{now.strftime('%Y%m%d')}-live",
|
||||
engineer="李工",
|
||||
project="lane",
|
||||
batch="demo_batch_live",
|
||||
started_at=active_start,
|
||||
status="active",
|
||||
source="mock",
|
||||
note="演示进行中趟次(长沙弯道)",
|
||||
)
|
||||
db.add(active)
|
||||
db.flush()
|
||||
# 进行中趟次也写入完整弯道轨迹(不再只种前半段)
|
||||
am = _add_points(db, active.id, coords, active_start)
|
||||
active.mileage_km = round(am, 3)
|
||||
v.last_lat = coords[-1][0]
|
||||
v.last_lng = coords[-1][1]
|
||||
v.last_ts = active_start + timedelta(seconds=(len(coords) - 1) * 12)
|
||||
v.set_meta({"sim_index": len(coords) - 1, "sim_route": device_id, "sim_dir": -1})
|
||||
db.add(FleetRunMilestone(run_id=active.id, type="start", name="出发", lat=coords[0][0], lng=coords[0][1], mileage_km=0.0, occurred_at=active_start))
|
||||
db.flush()
|
||||
return True
|
||||
|
||||
|
||||
def reseed_demo_fleet(db: Session) -> bool:
|
||||
"""清空并重新注入长沙演示数据。"""
|
||||
clear_demo_fleet(db)
|
||||
return seed_demo_fleet(db)
|
||||
544
platform/as_platform/fleet/service.py
Normal file
544
platform/as_platform/fleet/service.py
Normal file
@@ -0,0 +1,544 @@
|
||||
"""Fleet map business logic."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import desc
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from as_platform.db.models import (
|
||||
FleetCollectionRun,
|
||||
FleetRunMilestone,
|
||||
FleetTrackPoint,
|
||||
FleetVehicle,
|
||||
)
|
||||
from as_platform.fleet.geo import haversine_km, simplify_coords
|
||||
from as_platform.fleet.mock_seed import ROUTES
|
||||
from as_platform.redis.bus import publish
|
||||
|
||||
RUN_IDLE_TIMEOUT_MIN = 10
|
||||
|
||||
|
||||
def _parse_ts(ts: str | datetime | None) -> datetime:
|
||||
if ts is None:
|
||||
return datetime.now(timezone.utc)
|
||||
if isinstance(ts, datetime):
|
||||
return ts if ts.tzinfo else ts.replace(tzinfo=timezone.utc)
|
||||
s = str(ts).strip()
|
||||
if s.endswith("Z"):
|
||||
s = s[:-1] + "+00:00"
|
||||
return datetime.fromisoformat(s)
|
||||
|
||||
|
||||
def get_vehicle_by_device(db: Session, device_id: str) -> FleetVehicle | None:
|
||||
return db.query(FleetVehicle).filter_by(tbox_device_id=device_id).first()
|
||||
|
||||
|
||||
def get_active_run(db: Session, vehicle_id: int) -> FleetCollectionRun | None:
|
||||
return (
|
||||
db.query(FleetCollectionRun)
|
||||
.filter_by(vehicle_id=vehicle_id, status="active")
|
||||
.order_by(desc(FleetCollectionRun.id))
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def recalc_run_mileage(db: Session, run_id: int) -> float:
|
||||
pts = (
|
||||
db.query(FleetTrackPoint)
|
||||
.filter_by(run_id=run_id)
|
||||
.order_by(FleetTrackPoint.ts.asc())
|
||||
.all()
|
||||
)
|
||||
total = 0.0
|
||||
prev = None
|
||||
for p in pts:
|
||||
if prev:
|
||||
total += haversine_km(prev[0], prev[1], p.lat, p.lng)
|
||||
prev = (p.lat, p.lng)
|
||||
run = db.get(FleetCollectionRun, run_id)
|
||||
if run:
|
||||
run.mileage_km = round(total, 3)
|
||||
return total
|
||||
|
||||
|
||||
def _ensure_start_milestone(db: Session, run: FleetCollectionRun, lat: float, lng: float, ts: datetime) -> None:
|
||||
exists = db.query(FleetRunMilestone).filter_by(run_id=run.id, type="start").first()
|
||||
if exists:
|
||||
return
|
||||
db.add(FleetRunMilestone(run_id=run.id, type="start", name="出发", lat=lat, lng=lng, mileage_km=0.0, occurred_at=ts))
|
||||
|
||||
|
||||
def _close_run(db: Session, run: FleetCollectionRun, lat: float, lng: float, ts: datetime) -> None:
|
||||
run.status = "ended"
|
||||
run.ended_at = ts
|
||||
recalc_run_mileage(db, run.id)
|
||||
end_ms = db.query(FleetRunMilestone).filter_by(run_id=run.id, type="end").first()
|
||||
if not end_ms:
|
||||
db.add(
|
||||
FleetRunMilestone(
|
||||
run_id=run.id,
|
||||
type="end",
|
||||
name="结束",
|
||||
lat=lat,
|
||||
lng=lng,
|
||||
mileage_km=run.mileage_km,
|
||||
occurred_at=ts,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def ingest_tbox_gps(db: Session, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
device_id = str(payload.get("device_id") or "").strip()
|
||||
if not device_id:
|
||||
raise ValueError("device_id 必填")
|
||||
lat = float(payload["lat"])
|
||||
lng = float(payload["lng"])
|
||||
ts = _parse_ts(payload.get("ts"))
|
||||
speed = float(payload["speed_kmh"]) if payload.get("speed_kmh") is not None else None
|
||||
heading = float(payload["heading"]) if payload.get("heading") is not None else None
|
||||
run_signal = str(payload.get("run_signal") or "active").lower()
|
||||
plate_no = str(payload.get("plate_no") or "").strip()
|
||||
|
||||
vehicle = get_vehicle_by_device(db, device_id)
|
||||
if not vehicle:
|
||||
vehicle = FleetVehicle(
|
||||
plate_no=plate_no or device_id,
|
||||
tbox_device_id=device_id,
|
||||
name=plate_no or device_id,
|
||||
team="T-Box",
|
||||
status="active",
|
||||
online=True,
|
||||
)
|
||||
db.add(vehicle)
|
||||
db.flush()
|
||||
|
||||
vehicle.last_lat = lat
|
||||
vehicle.last_lng = lng
|
||||
vehicle.last_speed_kmh = speed
|
||||
vehicle.last_ts = ts
|
||||
vehicle.online = True
|
||||
|
||||
run = get_active_run(db, vehicle.id)
|
||||
if run_signal in ("idle", "off", "end", "stopped"):
|
||||
if run:
|
||||
_close_run(db, run, lat, lng, ts)
|
||||
vehicle.online = False
|
||||
db.flush()
|
||||
publish("fleet.gps", {"vehicle_id": vehicle.id, "device_id": device_id, "lat": lat, "lng": lng})
|
||||
return {"ok": True, "vehicle": vehicle.to_dict(), "run": run.to_dict() if run else None, "closed": True}
|
||||
|
||||
if not run:
|
||||
run_no = f"{device_id}-{ts.strftime('%Y%m%d%H%M')}"
|
||||
run = FleetCollectionRun(
|
||||
vehicle_id=vehicle.id,
|
||||
run_no=run_no,
|
||||
engineer=payload.get("engineer"),
|
||||
project=payload.get("project"),
|
||||
batch=payload.get("batch"),
|
||||
started_at=ts,
|
||||
status="active",
|
||||
source="tbox",
|
||||
)
|
||||
db.add(run)
|
||||
db.flush()
|
||||
_ensure_start_milestone(db, run, lat, lng, ts)
|
||||
|
||||
prev = (
|
||||
db.query(FleetTrackPoint)
|
||||
.filter_by(run_id=run.id)
|
||||
.order_by(desc(FleetTrackPoint.ts))
|
||||
.first()
|
||||
)
|
||||
seg = 0.0
|
||||
if prev:
|
||||
seg = haversine_km(prev.lat, prev.lng, lat, lng)
|
||||
db.add(FleetTrackPoint(run_id=run.id, ts=ts, lat=lat, lng=lng, speed_kmh=speed, heading=heading))
|
||||
run.mileage_km = round((run.mileage_km or 0.0) + seg, 3)
|
||||
db.flush()
|
||||
publish("fleet.gps", {"vehicle_id": vehicle.id, "run_id": run.id, "lat": lat, "lng": lng})
|
||||
return {"ok": True, "vehicle": vehicle.to_dict(), "run": run.to_dict()}
|
||||
|
||||
|
||||
def list_vehicles(db: Session) -> list[dict]:
|
||||
return [v.to_dict() for v in db.query(FleetVehicle).order_by(FleetVehicle.id).all()]
|
||||
|
||||
|
||||
def get_vehicle(db: Session, vehicle_id: int) -> dict | None:
|
||||
v = db.get(FleetVehicle, vehicle_id)
|
||||
return v.to_dict() if v else None
|
||||
|
||||
|
||||
def create_vehicle(db: Session, data: dict[str, Any]) -> dict:
|
||||
device_id = str(data.get("tbox_device_id") or "").strip()
|
||||
plate_no = str(data.get("plate_no") or "").strip()
|
||||
if not device_id:
|
||||
raise ValueError("tbox_device_id 必填")
|
||||
if not plate_no:
|
||||
raise ValueError("plate_no 必填")
|
||||
if get_vehicle_by_device(db, device_id):
|
||||
raise ValueError(f"设备 ID 已存在: {device_id}")
|
||||
v = FleetVehicle(
|
||||
plate_no=plate_no,
|
||||
tbox_device_id=device_id,
|
||||
name=str(data.get("name") or plate_no).strip(),
|
||||
team=str(data.get("team") or "").strip() or None,
|
||||
status=str(data.get("status") or "active").strip() or "active",
|
||||
online=False,
|
||||
)
|
||||
db.add(v)
|
||||
db.flush()
|
||||
return v.to_dict()
|
||||
|
||||
|
||||
def update_vehicle(db: Session, vehicle_id: int, data: dict[str, Any]) -> dict:
|
||||
v = db.get(FleetVehicle, vehicle_id)
|
||||
if not v:
|
||||
raise ValueError("车辆不存在")
|
||||
if "tbox_device_id" in data and data["tbox_device_id"]:
|
||||
new_dev = str(data["tbox_device_id"]).strip()
|
||||
other = get_vehicle_by_device(db, new_dev)
|
||||
if other and other.id != vehicle_id:
|
||||
raise ValueError(f"设备 ID 已被占用: {new_dev}")
|
||||
v.tbox_device_id = new_dev
|
||||
if "plate_no" in data and data["plate_no"]:
|
||||
v.plate_no = str(data["plate_no"]).strip()
|
||||
if "name" in data:
|
||||
v.name = str(data["name"] or v.plate_no).strip()
|
||||
if "team" in data:
|
||||
v.team = str(data["team"]).strip() or None
|
||||
if "status" in data and data["status"]:
|
||||
v.status = str(data["status"]).strip()
|
||||
db.flush()
|
||||
return v.to_dict()
|
||||
|
||||
|
||||
def delete_vehicle(db: Session, vehicle_id: int) -> None:
|
||||
v = db.get(FleetVehicle, vehicle_id)
|
||||
if not v:
|
||||
raise ValueError("车辆不存在")
|
||||
if get_active_run(db, vehicle_id):
|
||||
raise ValueError("该车有进行中的趟次,请先结束趟次后再删除")
|
||||
db.delete(v)
|
||||
|
||||
|
||||
def _run_duration_sec(run: FleetCollectionRun) -> int:
|
||||
if not run.started_at:
|
||||
return 0
|
||||
end = run.ended_at
|
||||
if end is None and run.status == "active":
|
||||
end = datetime.now(timezone.utc)
|
||||
if end is None:
|
||||
return 0
|
||||
return max(0, int((end - run.started_at).total_seconds()))
|
||||
|
||||
|
||||
def list_runs(
|
||||
db: Session,
|
||||
vehicle_id: int | None = None,
|
||||
status: str | None = None,
|
||||
*,
|
||||
offset: int = 0,
|
||||
limit: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
q = db.query(FleetCollectionRun).order_by(desc(FleetCollectionRun.started_at))
|
||||
if vehicle_id:
|
||||
q = q.filter_by(vehicle_id=vehicle_id)
|
||||
if status:
|
||||
q = q.filter_by(status=status)
|
||||
total = q.count()
|
||||
rows = q.offset(max(0, offset)).limit(max(1, limit)).all()
|
||||
out: list[dict] = []
|
||||
for r in rows:
|
||||
d = r.to_dict()
|
||||
d["point_count"] = db.query(FleetTrackPoint).filter_by(run_id=r.id).count()
|
||||
d["milestone_count"] = db.query(FleetRunMilestone).filter_by(run_id=r.id).count()
|
||||
d["duration_sec"] = _run_duration_sec(r)
|
||||
out.append(d)
|
||||
return {"items": out, "total": total, "offset": offset, "limit": limit}
|
||||
|
||||
|
||||
def get_run_detail(db: Session, run_id: int) -> dict | None:
|
||||
run = db.get(FleetCollectionRun, run_id)
|
||||
if not run:
|
||||
return None
|
||||
vehicle = db.get(FleetVehicle, run.vehicle_id)
|
||||
milestones = (
|
||||
db.query(FleetRunMilestone).filter_by(run_id=run_id).order_by(FleetRunMilestone.occurred_at.asc()).all()
|
||||
)
|
||||
return {
|
||||
"run": run.to_dict(),
|
||||
"vehicle": vehicle.to_dict() if vehicle else None,
|
||||
"milestones": [m.to_dict() for m in milestones],
|
||||
}
|
||||
|
||||
|
||||
def close_idle_runs(db: Session) -> int:
|
||||
"""无 GPS 超过 RUN_IDLE_TIMEOUT_MIN 的 active 趟次自动结束。"""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=RUN_IDLE_TIMEOUT_MIN)
|
||||
closed = 0
|
||||
for run in db.query(FleetCollectionRun).filter_by(status="active").all():
|
||||
vehicle = db.get(FleetVehicle, run.vehicle_id)
|
||||
last_ts = vehicle.last_ts if vehicle else None
|
||||
if not last_ts or last_ts >= cutoff:
|
||||
continue
|
||||
lat = float(vehicle.last_lat or 0) if vehicle else 0.0
|
||||
lng = float(vehicle.last_lng or 0) if vehicle else 0.0
|
||||
_close_run(db, run, lat, lng, datetime.now(timezone.utc))
|
||||
if vehicle:
|
||||
vehicle.online = False
|
||||
closed += 1
|
||||
return closed
|
||||
|
||||
|
||||
def ingest_tbox_gps_batch(db: Session, points: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
if not points:
|
||||
raise ValueError("points 不能为空")
|
||||
if len(points) > 100:
|
||||
raise ValueError("单次最多 100 个点")
|
||||
results: list[dict[str, Any]] = []
|
||||
for payload in points:
|
||||
results.append(ingest_tbox_gps(db, payload))
|
||||
return {"ok": True, "count": len(results), "last": results[-1] if results else None}
|
||||
|
||||
|
||||
def _parse_csv_track(content: str) -> list[tuple[float, float, datetime | None]]:
|
||||
import csv
|
||||
from io import StringIO
|
||||
|
||||
rows = list(csv.reader(StringIO(content.strip())))
|
||||
if not rows:
|
||||
return []
|
||||
start = 0
|
||||
header = [c.strip().lower() for c in rows[0]]
|
||||
if any(h in header for h in ("lat", "lng", "lon", "latitude", "longitude")):
|
||||
start = 1
|
||||
points: list[tuple[float, float, datetime | None]] = []
|
||||
for row in rows[start:]:
|
||||
if len(row) < 2:
|
||||
continue
|
||||
try:
|
||||
if start and len(header) >= 2:
|
||||
idx = {h: i for i, h in enumerate(header)}
|
||||
lat_i = idx.get("lat", idx.get("latitude", 0))
|
||||
lng_i = idx.get("lng", idx.get("lon", idx.get("longitude", 1)))
|
||||
lat = float(row[lat_i])
|
||||
lng = float(row[lng_i])
|
||||
ts_i = idx.get("ts") or idx.get("time")
|
||||
ts = _parse_ts(row[ts_i]) if ts_i is not None and ts_i < len(row) else None
|
||||
else:
|
||||
lat = float(row[0])
|
||||
lng = float(row[1])
|
||||
ts = _parse_ts(row[2]) if len(row) > 2 and row[2].strip() else None
|
||||
points.append((lat, lng, ts))
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
return points
|
||||
|
||||
|
||||
def import_csv_run(db: Session, *, vehicle_id: int, csv_content: str, note: str | None = None, project: str | None = None, batch: str | None = None) -> dict[str, Any]:
|
||||
vehicle = db.get(FleetVehicle, vehicle_id)
|
||||
if not vehicle:
|
||||
raise ValueError("车辆不存在")
|
||||
pts = _parse_csv_track(csv_content)
|
||||
if len(pts) < 2:
|
||||
raise ValueError("CSV 轨迹点不足")
|
||||
active = get_active_run(db, vehicle_id)
|
||||
if active:
|
||||
_close_run(db, active, pts[0][0], pts[0][1], pts[0][2] or datetime.now(timezone.utc))
|
||||
start_ts = pts[0][2] or datetime.now(timezone.utc)
|
||||
run_no = f"csv-{vehicle_id}-{int(start_ts.timestamp())}"
|
||||
run = FleetCollectionRun(
|
||||
vehicle_id=vehicle_id,
|
||||
run_no=run_no,
|
||||
status="ended",
|
||||
source="csv",
|
||||
note=note,
|
||||
project=project,
|
||||
batch=batch,
|
||||
started_at=start_ts,
|
||||
ended_at=pts[-1][2] or datetime.now(timezone.utc),
|
||||
)
|
||||
db.add(run)
|
||||
db.flush()
|
||||
for lat, lng, ts in pts:
|
||||
t = ts or start_ts
|
||||
db.add(FleetTrackPoint(run_id=run.id, lat=lat, lng=lng, ts=t, speed_kmh=0.0))
|
||||
_ensure_start_milestone(db, run, pts[0][0], pts[0][1], start_ts)
|
||||
_close_run(db, run, pts[-1][0], pts[-1][1], pts[-1][2] or datetime.now(timezone.utc))
|
||||
recalc_run_mileage(db, run.id)
|
||||
return {"ok": True, "run_id": run.id, "point_count": len(pts), "mileage_km": run.mileage_km}
|
||||
|
||||
|
||||
def get_live_fleet(db: Session) -> dict[str, Any]:
|
||||
close_idle_runs(db)
|
||||
vehicles = db.query(FleetVehicle).order_by(FleetVehicle.id).all()
|
||||
items = []
|
||||
for v in vehicles:
|
||||
active = get_active_run(db, v.id)
|
||||
items.append(
|
||||
{
|
||||
**v.to_dict(),
|
||||
"active_run_id": active.id if active else None,
|
||||
"active_mileage_km": active.mileage_km if active else None,
|
||||
"active_run_no": active.run_no if active else None,
|
||||
}
|
||||
)
|
||||
active_runs = db.query(FleetCollectionRun).filter_by(status="active").count()
|
||||
total_km = sum(r.mileage_km or 0.0 for r in db.query(FleetCollectionRun).all())
|
||||
return {
|
||||
"vehicles": items,
|
||||
"stats": {
|
||||
"vehicle_count": len(vehicles),
|
||||
"online_count": sum(1 for v in vehicles if v.online),
|
||||
"active_runs": active_runs,
|
||||
"total_mileage_km": round(total_km, 2),
|
||||
},
|
||||
"mock": True,
|
||||
}
|
||||
|
||||
|
||||
def get_run_track_geojson(db: Session, run_id: int, max_points: int = 5000) -> dict:
|
||||
pts = (
|
||||
db.query(FleetTrackPoint)
|
||||
.filter_by(run_id=run_id)
|
||||
.order_by(FleetTrackPoint.ts.asc())
|
||||
.all()
|
||||
)
|
||||
coords = [[p.lng, p.lat] for p in pts]
|
||||
coords = simplify_coords(coords, max_points)
|
||||
milestones = db.query(FleetRunMilestone).filter_by(run_id=run_id).all()
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"kind": "track", "run_id": run_id, "point_count": len(pts)},
|
||||
"geometry": {"type": "LineString", "coordinates": coords},
|
||||
},
|
||||
*[
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"kind": "milestone", **m.to_dict()},
|
||||
"geometry": {"type": "Point", "coordinates": [m.lng, m.lat]},
|
||||
}
|
||||
for m in milestones
|
||||
],
|
||||
],
|
||||
"points": [p.to_dict() for p in pts[-200:]],
|
||||
}
|
||||
|
||||
|
||||
def fleet_summary(db: Session) -> dict:
|
||||
runs = db.query(FleetCollectionRun).all()
|
||||
by_vehicle: dict[int, float] = {}
|
||||
for r in runs:
|
||||
by_vehicle[r.vehicle_id] = by_vehicle.get(r.vehicle_id, 0.0) + (r.mileage_km or 0.0)
|
||||
vehicles = {v.id: v for v in db.query(FleetVehicle).all()}
|
||||
per_vehicle = [
|
||||
{
|
||||
"vehicle_id": vid,
|
||||
"plate_no": vehicles[vid].plate_no if vid in vehicles else str(vid),
|
||||
"mileage_km": round(km, 2),
|
||||
}
|
||||
for vid, km in by_vehicle.items()
|
||||
]
|
||||
return {
|
||||
"total_runs": len(runs),
|
||||
"active_runs": sum(1 for r in runs if r.status == "active"),
|
||||
"total_mileage_km": round(sum(r.mileage_km or 0.0 for r in runs), 2),
|
||||
"per_vehicle": per_vehicle,
|
||||
}
|
||||
|
||||
|
||||
def simulate_tick(db: Session) -> int:
|
||||
"""Advance mock vehicles along ROUTES (demo)."""
|
||||
n = 0
|
||||
now = datetime.now(timezone.utc)
|
||||
for v in db.query(FleetVehicle).all():
|
||||
meta = v.meta()
|
||||
route_key = meta.get("sim_route") or v.tbox_device_id
|
||||
coords = ROUTES.get(route_key)
|
||||
if not coords:
|
||||
continue
|
||||
run = get_active_run(db, v.id)
|
||||
if not run:
|
||||
continue
|
||||
idx = int(meta.get("sim_index", 0))
|
||||
direction = int(meta.get("sim_dir", 1))
|
||||
next_idx = idx + direction
|
||||
if next_idx >= len(coords):
|
||||
next_idx = len(coords) - 2
|
||||
direction = -1
|
||||
elif next_idx < 0:
|
||||
next_idx = 1
|
||||
direction = 1
|
||||
lat, lng = coords[next_idx]
|
||||
payload = {
|
||||
"device_id": v.tbox_device_id,
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"ts": now.isoformat(),
|
||||
"speed_kmh": 38.0 + (next_idx % 4),
|
||||
"run_signal": "active",
|
||||
}
|
||||
ingest_tbox_gps(db, payload)
|
||||
v.set_meta({**meta, "sim_index": next_idx, "sim_route": route_key, "sim_dir": direction})
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def _parse_gpx_points(content: str) -> list[tuple[float, float, datetime | None]]:
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
root = ET.fromstring(content)
|
||||
ns = ""
|
||||
if root.tag.startswith("{"):
|
||||
ns = root.tag.split("}")[0] + "}"
|
||||
points: list[tuple[float, float, datetime | None]] = []
|
||||
for trkpt in root.iter(f"{ns}trkpt"):
|
||||
lat_s = trkpt.get("lat")
|
||||
lon_s = trkpt.get("lon")
|
||||
if lat_s is None or lon_s is None:
|
||||
continue
|
||||
ts_el = trkpt.find(f"{ns}time")
|
||||
ts = _parse_ts(ts_el.text) if ts_el is not None and ts_el.text else None
|
||||
points.append((float(lat_s), float(lon_s), ts))
|
||||
return points
|
||||
|
||||
|
||||
def import_gpx_run(db: Session, *, vehicle_id: int, gpx_content: str, note: str | None = None) -> dict[str, Any]:
|
||||
vehicle = db.get(FleetVehicle, vehicle_id)
|
||||
if not vehicle:
|
||||
raise ValueError("车辆不存在")
|
||||
pts = _parse_gpx_points(gpx_content)
|
||||
if len(pts) < 2:
|
||||
raise ValueError("GPX 轨迹点不足")
|
||||
active = get_active_run(db, vehicle_id)
|
||||
if active:
|
||||
_close_run(db, active, pts[0][0], pts[0][1], pts[0][2] or datetime.now(timezone.utc))
|
||||
start_ts = pts[0][2] or datetime.now(timezone.utc)
|
||||
run_no = f"gpx-{vehicle_id}-{int(start_ts.timestamp())}"
|
||||
run = FleetCollectionRun(
|
||||
vehicle_id=vehicle_id,
|
||||
run_no=run_no,
|
||||
status="ended",
|
||||
source="gpx",
|
||||
note=note,
|
||||
started_at=start_ts,
|
||||
ended_at=pts[-1][2] or datetime.now(timezone.utc),
|
||||
)
|
||||
db.add(run)
|
||||
db.flush()
|
||||
prev = None
|
||||
for lat, lng, ts in pts:
|
||||
t = ts or start_ts
|
||||
db.add(FleetTrackPoint(run_id=run.id, lat=lat, lng=lng, ts=t, speed_kmh=0.0))
|
||||
if prev is None:
|
||||
_ensure_start_milestone(db, run, lat, lng, t)
|
||||
prev = (lat, lng)
|
||||
end_ts = pts[-1][2] or datetime.now(timezone.utc)
|
||||
_close_run(db, run, pts[-1][0], pts[-1][1], end_ts)
|
||||
recalc_run_mileage(db, run.id)
|
||||
return {"ok": True, "run_id": run.id, "point_count": len(pts), "mileage_km": run.mileage_km}
|
||||
1
platform/as_platform/integrations/__init__.py
Normal file
1
platform/as_platform/integrations/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""外部系统集成。"""
|
||||
120
platform/as_platform/integrations/delivery_ingest.py
Normal file
120
platform/as_platform/integrations/delivery_ingest.py
Normal file
@@ -0,0 +1,120 @@
|
||||
"""平台送标申请审批通过后:analyze → promote 入湖。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.data.lake import (
|
||||
analyze_directory_candidate,
|
||||
create_directory_candidate,
|
||||
promote_candidate_to_inbox,
|
||||
)
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import BatchDelivery
|
||||
|
||||
|
||||
def validate_delivery_fields(
|
||||
*,
|
||||
project: str,
|
||||
task: str | None,
|
||||
mode: str | None,
|
||||
batch_name: str,
|
||||
data_path: str,
|
||||
) -> str | None:
|
||||
if not project or not batch_name:
|
||||
return "缺少 项目 或 批次名"
|
||||
if batch_name == (task or ""):
|
||||
return "批次名不能与任务名相同"
|
||||
if task in ("dam", "forward") and not (mode or "").strip():
|
||||
return f"任务 {task} 须填写 子模式"
|
||||
path = (data_path or "").strip()
|
||||
if not path:
|
||||
return "须填写 数据路径"
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return f"数据路径不存在: {path}"
|
||||
if project == "dms" and p.name in ("train", "val", "test") and p.parent.name == "images":
|
||||
return f"请填批次根目录(例如 {p.parent.parent}),不要填到 images/train"
|
||||
return None
|
||||
|
||||
|
||||
def ingest_from_directory(
|
||||
*,
|
||||
project: str,
|
||||
task: str | None,
|
||||
mode: str | None,
|
||||
batch_name: str,
|
||||
data_path: str,
|
||||
delivery_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""目录分析并 promote 到 inbox。"""
|
||||
err = validate_delivery_fields(
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
batch_name=batch_name,
|
||||
data_path=data_path,
|
||||
)
|
||||
if err:
|
||||
raise ValueError(err)
|
||||
|
||||
src = Path(data_path.strip())
|
||||
task_eff = task if project == "dms" else None
|
||||
cand = create_directory_candidate(
|
||||
project=project,
|
||||
task=task_eff,
|
||||
mode=mode or None,
|
||||
source_dir=src,
|
||||
source_type="platform_delivery",
|
||||
external_id=delivery_id,
|
||||
)
|
||||
cid = cand["id"]
|
||||
analyze_directory_candidate(cid, src)
|
||||
promo = promote_candidate_to_inbox(cid, batch=batch_name, mode=mode or None)
|
||||
return {
|
||||
"ok": True,
|
||||
"candidate_id": cid,
|
||||
"inbox_path": promo.get("inbox_path") or "",
|
||||
"batch": promo.get("batch") or batch_name,
|
||||
}
|
||||
|
||||
|
||||
def run_delivery_ingest(delivery_id: str) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchDelivery, delivery_id)
|
||||
if not rec:
|
||||
raise ValueError(f"送标申请不存在: {delivery_id}")
|
||||
rec.status = "ingesting"
|
||||
rec.error_message = None
|
||||
db.flush()
|
||||
project = rec.project
|
||||
task = rec.task
|
||||
mode = rec.mode
|
||||
batch_name = rec.batch_name
|
||||
data_path = rec.data_path
|
||||
|
||||
try:
|
||||
result = ingest_from_directory(
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
batch_name=batch_name,
|
||||
data_path=data_path,
|
||||
delivery_id=delivery_id,
|
||||
)
|
||||
except Exception as e:
|
||||
from as_platform.deliveries.service import mark_delivery_ingest_failed
|
||||
|
||||
mark_delivery_ingest_failed(delivery_id, None, str(e))
|
||||
raise
|
||||
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchDelivery, delivery_id)
|
||||
if rec:
|
||||
rec.status = "in_lake"
|
||||
rec.candidate_id = result.get("candidate_id")
|
||||
rec.inbox_path = result.get("inbox_path")
|
||||
rec.error_message = None
|
||||
db.flush()
|
||||
|
||||
return result
|
||||
299
platform/as_platform/integrations/feishu_bitable.py
Normal file
299
platform/as_platform/integrations/feishu_bitable.py
Normal file
@@ -0,0 +1,299 @@
|
||||
"""飞书多维表格 API(tenant_access_token)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from as_platform.auth.feishu import _get_tenant_access_token
|
||||
from as_platform.config import (
|
||||
FEISHU_APP_ID,
|
||||
FEISHU_APP_SECRET,
|
||||
FEISHU_BITABLE_APP_TOKEN,
|
||||
FEISHU_BITABLE_FIELDS,
|
||||
FEISHU_BITABLE_TABLE_ID,
|
||||
FEISHU_BITABLE_WIKI_NODE_TOKEN,
|
||||
)
|
||||
|
||||
BITABLE_BASE = "https://open.feishu.cn/open-apis/bitable/v1"
|
||||
WIKI_GET_NODE = "https://open.feishu.cn/open-apis/wiki/v2/spaces/get_node"
|
||||
|
||||
_field_cache: dict[str, Any] | None = None
|
||||
_field_cache_at: float = 0.0
|
||||
_FIELD_CACHE_TTL = 300.0
|
||||
_resolved_app_token: str | None = None
|
||||
|
||||
|
||||
def _looks_like_bitable_app_token(token: str) -> bool:
|
||||
t = token.strip()
|
||||
return t.startswith(("Basc", "basc", "app"))
|
||||
|
||||
|
||||
def _resolve_app_token_from_wiki(node_token: str) -> str:
|
||||
with _client() as client:
|
||||
access = _token(client)
|
||||
resp = client.get(
|
||||
WIKI_GET_NODE,
|
||||
headers={"Authorization": f"Bearer {access}"},
|
||||
params={"token": node_token},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "wiki get_node failed")
|
||||
node = data.get("data", {}).get("node") or {}
|
||||
obj = (node.get("obj_token") or "").strip()
|
||||
if not obj:
|
||||
raise RuntimeError(
|
||||
"wiki 节点未返回 obj_token;请在开放平台开通 wiki:node:read,或改用 /base/Basc... 填 FEISHU_BITABLE_APP_TOKEN"
|
||||
)
|
||||
return obj
|
||||
|
||||
|
||||
def _effective_app_token() -> str:
|
||||
global _resolved_app_token
|
||||
if FEISHU_BITABLE_APP_TOKEN and _looks_like_bitable_app_token(FEISHU_BITABLE_APP_TOKEN):
|
||||
return FEISHU_BITABLE_APP_TOKEN
|
||||
node = FEISHU_BITABLE_WIKI_NODE_TOKEN or FEISHU_BITABLE_APP_TOKEN
|
||||
if not node:
|
||||
return FEISHU_BITABLE_APP_TOKEN
|
||||
if _resolved_app_token is None:
|
||||
_resolved_app_token = _resolve_app_token_from_wiki(node)
|
||||
return _resolved_app_token
|
||||
|
||||
|
||||
def is_bitable_configured() -> bool:
|
||||
has_app = bool(FEISHU_BITABLE_APP_TOKEN or FEISHU_BITABLE_WIKI_NODE_TOKEN)
|
||||
return bool(FEISHU_APP_ID and FEISHU_APP_SECRET and has_app and FEISHU_BITABLE_TABLE_ID)
|
||||
|
||||
|
||||
def _client() -> httpx.Client:
|
||||
return httpx.Client(timeout=60.0)
|
||||
|
||||
|
||||
def _token(client: httpx.Client) -> str:
|
||||
return _get_tenant_access_token(client)
|
||||
|
||||
|
||||
def _app_table() -> tuple[str, str]:
|
||||
return _effective_app_token(), FEISHU_BITABLE_TABLE_ID
|
||||
|
||||
|
||||
def list_tables() -> list[dict[str, Any]]:
|
||||
"""列出 Base 下数据表(运维查 table_id)。"""
|
||||
if not is_bitable_configured():
|
||||
return []
|
||||
app_token, _ = _app_table()
|
||||
with _client() as client:
|
||||
token = _token(client)
|
||||
resp = client.get(
|
||||
f"{BITABLE_BASE}/apps/{app_token}/tables",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
params={"page_size": 100},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "list tables failed")
|
||||
return data.get("data", {}).get("items") or []
|
||||
|
||||
|
||||
def _load_field_meta(force: bool = False) -> dict[str, dict[str, Any]]:
|
||||
global _field_cache, _field_cache_at
|
||||
now = time.time()
|
||||
if not force and _field_cache and now - _field_cache_at < _FIELD_CACHE_TTL:
|
||||
return _field_cache
|
||||
|
||||
app_token, table_id = _app_table()
|
||||
by_name: dict[str, dict[str, Any]] = {}
|
||||
with _client() as client:
|
||||
token = _token(client)
|
||||
page_token: str | None = None
|
||||
while True:
|
||||
params: dict[str, Any] = {"page_size": 100}
|
||||
if page_token:
|
||||
params["page_token"] = page_token
|
||||
resp = client.get(
|
||||
f"{BITABLE_BASE}/apps/{app_token}/tables/{table_id}/fields",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
params=params,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "list fields failed")
|
||||
for item in data.get("data", {}).get("items") or []:
|
||||
name = item.get("field_name") or item.get("name")
|
||||
if name:
|
||||
by_name[str(name)] = item
|
||||
page_token = data.get("data", {}).get("page_token")
|
||||
if not page_token:
|
||||
break
|
||||
_field_cache = by_name
|
||||
_field_cache_at = now
|
||||
return by_name
|
||||
|
||||
|
||||
def _option_id_for_text(field_meta: dict[str, Any], text: str) -> str | None:
|
||||
ui = field_meta.get("ui_type") or field_meta.get("type")
|
||||
if ui not in (3, "SingleSelect", "single_select"):
|
||||
return None
|
||||
prop = field_meta.get("property") or {}
|
||||
for opt in prop.get("options") or []:
|
||||
if opt.get("name") == text:
|
||||
return opt.get("id")
|
||||
return None
|
||||
|
||||
|
||||
def _encode_value(field_meta: dict[str, Any], value: Any) -> Any:
|
||||
if value is None:
|
||||
return None
|
||||
ui = field_meta.get("ui_type") or field_meta.get("type")
|
||||
if ui in (1, "Text", "text", "Url", "url"):
|
||||
if isinstance(value, dict) and "link" in value:
|
||||
return value
|
||||
return str(value)
|
||||
if ui in (2, "Number", "number"):
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if ui in (3, "SingleSelect", "single_select"):
|
||||
oid = _option_id_for_text(field_meta, str(value))
|
||||
return oid if oid else str(value)
|
||||
if ui in (5, "DateTime", "datetime", "CreatedTime", "ModifiedTime"):
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
if isinstance(value, datetime):
|
||||
dt = value if value.tzinfo else value.replace(tzinfo=timezone.utc)
|
||||
return int(dt.timestamp() * 1000)
|
||||
return int(datetime.now(timezone.utc).timestamp() * 1000)
|
||||
if ui in (15, "Url", "url") and isinstance(value, dict):
|
||||
return {"text": value.get("text") or value.get("link"), "link": value.get("link")}
|
||||
return str(value)
|
||||
|
||||
|
||||
def _flatten_cell(cell: Any) -> str:
|
||||
if cell is None:
|
||||
return ""
|
||||
if isinstance(cell, str):
|
||||
return cell.strip()
|
||||
if isinstance(cell, (int, float)):
|
||||
return str(cell)
|
||||
if isinstance(cell, list):
|
||||
parts = [_flatten_cell(x) for x in cell]
|
||||
return ",".join(p for p in parts if p)
|
||||
if isinstance(cell, dict):
|
||||
if "text" in cell:
|
||||
return str(cell.get("text") or "").strip()
|
||||
if "name" in cell:
|
||||
return str(cell.get("name") or "").strip()
|
||||
if "value" in cell:
|
||||
return _flatten_cell(cell.get("value"))
|
||||
return str(cell).strip()
|
||||
|
||||
|
||||
def list_all_records() -> list[dict[str, Any]]:
|
||||
"""返回 {record_id, fields_raw, flat}。"""
|
||||
if not is_bitable_configured():
|
||||
return []
|
||||
app_token, table_id = _app_table()
|
||||
meta = _load_field_meta()
|
||||
id_by_label = {label: meta[name]["field_id"] for name, m in meta.items() if (label := FEISHU_BITABLE_FIELDS.get(name)) and name in meta for name in FEISHU_BITABLE_FIELDS}
|
||||
# rebuild: map logical key -> field_name
|
||||
name_for_key = {k: v for k, v in FEISHU_BITABLE_FIELDS.items()}
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
with _client() as client:
|
||||
token = _token(client)
|
||||
page_token: str | None = None
|
||||
while True:
|
||||
body: dict[str, Any] = {"page_size": 500}
|
||||
if page_token:
|
||||
body["page_token"] = page_token
|
||||
resp = client.post(
|
||||
f"{BITABLE_BASE}/apps/{app_token}/tables/{table_id}/records/search",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json=body,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "search records failed")
|
||||
for item in data.get("data", {}).get("items") or []:
|
||||
record_id = item.get("record_id") or item.get("id")
|
||||
fields = item.get("fields") or {}
|
||||
flat: dict[str, str] = {}
|
||||
for key, col_name in name_for_key.items():
|
||||
raw = fields.get(col_name)
|
||||
if raw is None:
|
||||
fid = meta.get(col_name, {}).get("field_id")
|
||||
if fid:
|
||||
raw = fields.get(fid)
|
||||
flat[key] = _flatten_cell(raw)
|
||||
rows.append({"record_id": record_id, "fields": fields, "flat": flat})
|
||||
page_token = data.get("data", {}).get("page_token")
|
||||
if not page_token:
|
||||
break
|
||||
return rows
|
||||
|
||||
|
||||
def update_record(record_id: str, values: dict[str, Any]) -> None:
|
||||
"""values: 逻辑键 delivery_id / status / inbox_path ..."""
|
||||
if not record_id or not is_bitable_configured():
|
||||
return
|
||||
meta = _load_field_meta()
|
||||
payload_fields: dict[str, Any] = {}
|
||||
for key, val in values.items():
|
||||
if val is None:
|
||||
continue
|
||||
col = FEISHU_BITABLE_FIELDS.get(key)
|
||||
if not col or col not in meta:
|
||||
continue
|
||||
encoded = _encode_value(meta[col], val)
|
||||
if encoded is not None:
|
||||
payload_fields[col] = encoded
|
||||
|
||||
if not payload_fields:
|
||||
return
|
||||
|
||||
app_token, table_id = _app_table()
|
||||
with _client() as client:
|
||||
token = _token(client)
|
||||
resp = client.put(
|
||||
f"{BITABLE_BASE}/apps/{app_token}/tables/{table_id}/records/{record_id}",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json={"fields": payload_fields},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
raise RuntimeError(data.get("msg") or "update record failed")
|
||||
|
||||
|
||||
def connectivity_check() -> dict[str, Any]:
|
||||
if not is_bitable_configured():
|
||||
return {"configured": False, "ok": False, "message": "缺少 FEISHU_BITABLE_APP_TOKEN 或 TABLE_ID"}
|
||||
try:
|
||||
tables = list_tables()
|
||||
meta = _load_field_meta(force=True)
|
||||
missing = [v for k, v in FEISHU_BITABLE_FIELDS.items() if k not in ("record_id",) and v not in meta]
|
||||
return {
|
||||
"configured": True,
|
||||
"ok": len(missing) == 0,
|
||||
"tables": [{"table_id": t.get("table_id"), "name": t.get("name")} for t in tables],
|
||||
"field_count": len(meta),
|
||||
"missing_columns": missing,
|
||||
}
|
||||
except Exception as e:
|
||||
msg = str(e)
|
||||
hint = ""
|
||||
if "1254302" in msg or "no permissions" in msg.lower():
|
||||
hint = (
|
||||
"飞书返回 1254302:应用「主动安全算法平台」对该表无读写权。"
|
||||
"知识库内嵌表常无法加企业应用,请复制为独立多维表格(/base/Basc...)并加应用协作者,"
|
||||
"或见 docs/FEISHU_BITABLE_OPS.md §8;联调可设 FEISHU_BITABLE_SYNC_ENABLED=0。"
|
||||
)
|
||||
return {"configured": True, "ok": False, "message": msg, "hint": hint or None}
|
||||
158
platform/as_platform/integrations/feishu_bitable_ingest.py
Normal file
158
platform/as_platform/integrations/feishu_bitable_ingest.py
Normal file
@@ -0,0 +1,158 @@
|
||||
"""飞书多维表格「待落盘」→ analyze → promote(Phase B,FEISHU_BITABLE_AUTO_INGEST)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import FEISHU_BITABLE_FIELDS
|
||||
from as_platform.data.lake import (
|
||||
analyze_directory_candidate,
|
||||
create_feishu_directory_candidate,
|
||||
promote_candidate_to_inbox,
|
||||
)
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import DatasetCandidate, FeishuBitableLink
|
||||
from as_platform.integrations.feishu_bitable import is_bitable_configured, list_all_records, update_record
|
||||
from as_platform.integrations.feishu_bitable_sync import batch_key
|
||||
|
||||
|
||||
def _validate_row(flat: dict[str, str]) -> str | None:
|
||||
project = flat.get("project") or ""
|
||||
task = flat.get("task") or ""
|
||||
batch_name = flat.get("batch_name") or ""
|
||||
mode = flat.get("mode") or ""
|
||||
if not project or not batch_name:
|
||||
return "缺少 项目 或 批次名"
|
||||
if batch_name == task:
|
||||
return "批次名不能与任务名相同"
|
||||
if task in ("dam", "forward") and not mode:
|
||||
return f"任务 {task} 须填写 子模式"
|
||||
data_path = (flat.get("data_path") or "").strip()
|
||||
if not data_path:
|
||||
return "待落盘须填写 数据路径(内网 NAS)"
|
||||
if not Path(data_path).exists():
|
||||
return f"数据路径不存在: {data_path}"
|
||||
return None
|
||||
|
||||
|
||||
def _already_ingested(flat: dict[str, str], delivery_id: str) -> bool:
|
||||
cid = (flat.get("candidate_id") or "").strip()
|
||||
if cid:
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, cid)
|
||||
if rec and rec.status in ("promoted", "analyzed") and rec.inbox_path:
|
||||
return True
|
||||
if delivery_id:
|
||||
with session_scope() as db:
|
||||
link = db.query(FeishuBitableLink).filter_by(delivery_id=delivery_id).first()
|
||||
if link and link.inbox_path:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def process_pending_ingest() -> dict[str, Any]:
|
||||
if not is_bitable_configured():
|
||||
return {"ok": False, "message": "未配置多维表格", "processed": 0}
|
||||
|
||||
records = list_all_records()
|
||||
processed = 0
|
||||
skipped = 0
|
||||
errors: list[str] = []
|
||||
|
||||
for rec in records:
|
||||
record_id = rec.get("record_id")
|
||||
flat = rec.get("flat") or {}
|
||||
if not record_id:
|
||||
continue
|
||||
if flat.get("status") != "待落盘":
|
||||
continue
|
||||
|
||||
delivery_id = flat.get("delivery_id") or ""
|
||||
if _already_ingested(flat, delivery_id):
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
err = _validate_row(flat)
|
||||
if err:
|
||||
try:
|
||||
update_record(record_id, {"status": "落盘失败", "error_message": err})
|
||||
except Exception as e:
|
||||
errors.append(f"{record_id}: {e}")
|
||||
continue
|
||||
|
||||
try:
|
||||
update_record(record_id, {"status": "分析中", "error_message": ""})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
project = flat.get("project") or "dms"
|
||||
task = flat.get("task") or None
|
||||
mode = flat.get("mode") or None
|
||||
batch_name = flat.get("batch_name") or ""
|
||||
src = Path((flat.get("data_path") or "").strip())
|
||||
|
||||
try:
|
||||
cand = create_feishu_directory_candidate(
|
||||
project=project,
|
||||
task=task if project == "dms" else None,
|
||||
mode=mode or None,
|
||||
source_dir=src,
|
||||
external_id=delivery_id or None,
|
||||
feishu_record_id=record_id,
|
||||
)
|
||||
cid = cand["id"]
|
||||
analyze_directory_candidate(cid, src)
|
||||
promo = promote_candidate_to_inbox(cid, batch=batch_name, mode=mode or None)
|
||||
inbox_path = promo.get("inbox_path") or ""
|
||||
|
||||
update_record(
|
||||
record_id,
|
||||
{
|
||||
"status": "待送标",
|
||||
"candidate_id": cid,
|
||||
"inbox_path": inbox_path,
|
||||
"error_message": "",
|
||||
"record_id": record_id,
|
||||
},
|
||||
)
|
||||
|
||||
key = batch_key(
|
||||
project,
|
||||
task if project == "dms" else None,
|
||||
mode,
|
||||
promo.get("batch") or batch_name,
|
||||
)
|
||||
with session_scope() as db:
|
||||
link = db.query(FeishuBitableLink).filter_by(batch_key=key).first()
|
||||
if not link:
|
||||
link = FeishuBitableLink(
|
||||
batch_key=key,
|
||||
record_id=record_id,
|
||||
delivery_id=delivery_id or None,
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
batch=promo.get("batch") or batch_name,
|
||||
)
|
||||
db.add(link)
|
||||
link.record_id = record_id
|
||||
link.delivery_id = delivery_id or link.delivery_id
|
||||
link.inbox_path = inbox_path
|
||||
db.flush()
|
||||
|
||||
processed += 1
|
||||
except Exception as e:
|
||||
msg = str(e)
|
||||
errors.append(f"{record_id}: {msg}")
|
||||
try:
|
||||
update_record(record_id, {"status": "落盘失败", "error_message": msg[:500]})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"processed": processed,
|
||||
"skipped": skipped,
|
||||
"errors": errors[:20],
|
||||
"status_field": FEISHU_BITABLE_FIELDS.get("status"),
|
||||
}
|
||||
236
platform/as_platform/integrations/feishu_bitable_sync.py
Normal file
236
platform/as_platform/integrations/feishu_bitable_sync.py
Normal file
@@ -0,0 +1,236 @@
|
||||
"""HSAP 批次 → 飞书多维表格回写(内网 Phase A)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import FEISHU_STATUS_FROM_STAGE, FRONTEND_URL
|
||||
from as_platform.data.core import get_pending_report
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import FeishuBitableLink
|
||||
from as_platform.integrations.feishu_bitable import is_bitable_configured, list_all_records, update_record
|
||||
from as_platform.labeling.progress import campaign_progress_summary
|
||||
from as_platform.labeling.service import list_labeling_batches
|
||||
|
||||
|
||||
def batch_key(project: str, task: str | None, mode: str | None, batch: str, location: str = "inbox") -> str:
|
||||
return f"{location}:{project}:{task or ''}:{mode or ''}:{batch}"
|
||||
|
||||
|
||||
def _collect_hsap_batches() -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
try:
|
||||
for row in list_labeling_batches().get("items") or []:
|
||||
key = batch_key(
|
||||
row.get("project") or "dms",
|
||||
row.get("task"),
|
||||
row.get("mode"),
|
||||
row.get("batch") or "",
|
||||
row.get("location") or "inbox",
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
items.append(row)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
report = get_pending_report()
|
||||
for row in report.get("batches") or []:
|
||||
if row.get("stage") == "ingested":
|
||||
continue
|
||||
key = batch_key(
|
||||
row.get("project") or "dms",
|
||||
row.get("task"),
|
||||
row.get("mode"),
|
||||
row.get("batch") or "",
|
||||
row.get("location") or "inbox",
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
items.append(row)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _find_batch_for_record(
|
||||
flat: dict[str, str],
|
||||
hsap_batches: list[dict[str, Any]],
|
||||
links_by_delivery: dict[str, FeishuBitableLink],
|
||||
) -> dict[str, Any] | None:
|
||||
did = (flat.get("delivery_id") or "").strip()
|
||||
if did and did in links_by_delivery:
|
||||
key = links_by_delivery[did].batch_key
|
||||
for b in hsap_batches:
|
||||
if batch_key(
|
||||
b.get("project") or "dms",
|
||||
b.get("task"),
|
||||
b.get("mode"),
|
||||
b.get("batch") or "",
|
||||
b.get("location") or "inbox",
|
||||
) == key:
|
||||
return b
|
||||
for b in hsap_batches:
|
||||
if _match_record_to_batch(flat, b):
|
||||
return b
|
||||
return None
|
||||
|
||||
|
||||
def _match_record_to_batch(flat: dict[str, str], b: dict[str, Any]) -> bool:
|
||||
if flat.get("project") and flat.get("project") != (b.get("project") or ""):
|
||||
return False
|
||||
if flat.get("task") and flat.get("task") != (b.get("task") or ""):
|
||||
return False
|
||||
bm = flat.get("batch_name") or ""
|
||||
if bm and bm != (b.get("batch") or ""):
|
||||
return False
|
||||
fm = flat.get("mode") or ""
|
||||
if fm and fm != (b.get("mode") or ""):
|
||||
return False
|
||||
if not bm and not fm:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _progress_for_batch(b: dict[str, Any]) -> tuple[str, str | None]:
|
||||
cid = b.get("campaign_id")
|
||||
if not cid:
|
||||
return "", None
|
||||
try:
|
||||
prog = campaign_progress_summary(cid)
|
||||
return f"{prog.get('completed_tasks', 0)}/{prog.get('total_tasks', 0)}", cid
|
||||
except Exception:
|
||||
return "", cid
|
||||
|
||||
|
||||
def _hsap_link(b: dict[str, Any], campaign_id: str | None) -> dict[str, str]:
|
||||
base = FRONTEND_URL.rstrip("/")
|
||||
if campaign_id:
|
||||
path = f"/labeling/campaigns/{campaign_id}/annotate"
|
||||
label = "进入标注"
|
||||
else:
|
||||
path = "/labeling"
|
||||
label = "送标工作台"
|
||||
url = f"{base}{path}"
|
||||
batch = b.get("batch") or ""
|
||||
return {"text": f"{label} · {batch}", "link": url}
|
||||
|
||||
|
||||
def sync_hsap_to_bitable() -> dict[str, Any]:
|
||||
if not is_bitable_configured():
|
||||
return {"ok": False, "message": "飞书多维表格未配置", "updated": 0}
|
||||
|
||||
hsap_batches = _collect_hsap_batches()
|
||||
records = list_all_records()
|
||||
updated = 0
|
||||
matched_keys: set[str] = set()
|
||||
errors: list[str] = []
|
||||
|
||||
with session_scope() as db:
|
||||
links = db.query(FeishuBitableLink).all()
|
||||
links_by_delivery = {lnk.delivery_id: lnk for lnk in links if lnk.delivery_id}
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
for rec in records:
|
||||
record_id = rec.get("record_id")
|
||||
flat = rec.get("flat") or {}
|
||||
if not record_id:
|
||||
continue
|
||||
|
||||
hit = _find_batch_for_record(flat, hsap_batches, links_by_delivery)
|
||||
if not hit:
|
||||
continue
|
||||
|
||||
key = batch_key(
|
||||
hit.get("project") or "dms",
|
||||
hit.get("task"),
|
||||
hit.get("mode"),
|
||||
hit.get("batch") or "",
|
||||
hit.get("location") or "inbox",
|
||||
)
|
||||
matched_keys.add(key)
|
||||
|
||||
stage = hit.get("stage") or "raw_pool"
|
||||
feishu_status = FEISHU_STATUS_FROM_STAGE.get(stage)
|
||||
progress, cid = _progress_for_batch(hit)
|
||||
path = hit.get("path") or ""
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"record_id": record_id,
|
||||
"inbox_path": path,
|
||||
"progress": progress or None,
|
||||
"campaign_id": cid,
|
||||
"hsap_link": _hsap_link(hit, cid),
|
||||
"last_sync": now,
|
||||
}
|
||||
if feishu_status:
|
||||
payload["status"] = feishu_status
|
||||
|
||||
try:
|
||||
update_record(record_id, payload)
|
||||
updated += 1
|
||||
with session_scope() as db:
|
||||
link = db.query(FeishuBitableLink).filter_by(batch_key=key).first()
|
||||
if not link:
|
||||
link = FeishuBitableLink(
|
||||
batch_key=key,
|
||||
record_id=record_id,
|
||||
project=hit.get("project") or "dms",
|
||||
task=hit.get("task"),
|
||||
mode=hit.get("mode"),
|
||||
batch=hit.get("batch") or "",
|
||||
)
|
||||
db.add(link)
|
||||
link.record_id = record_id
|
||||
link.delivery_id = flat.get("delivery_id") or link.delivery_id
|
||||
link.campaign_id = cid
|
||||
link.inbox_path = path
|
||||
link.last_sync_at = now
|
||||
db.flush()
|
||||
except Exception as e:
|
||||
errors.append(f"{record_id}: {e}")
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"updated": updated,
|
||||
"hsap_batches": len(hsap_batches),
|
||||
"bitable_rows": len(records),
|
||||
"matched": len(matched_keys),
|
||||
"errors": errors[:20],
|
||||
}
|
||||
|
||||
|
||||
def backfill_hints() -> dict[str, Any]:
|
||||
"""返回尚未在飞书表匹配到的 HSAP 批次(需人工补录行)。"""
|
||||
hsap_batches = _collect_hsap_batches()
|
||||
records = list_all_records() if is_bitable_configured() else []
|
||||
unmatched: list[dict[str, Any]] = []
|
||||
|
||||
for b in hsap_batches:
|
||||
found = False
|
||||
for rec in records:
|
||||
if _match_record_to_batch(rec.get("flat") or {}, b):
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
unmatched.append(
|
||||
{
|
||||
"project": b.get("project"),
|
||||
"task": b.get("task"),
|
||||
"mode": b.get("mode"),
|
||||
"batch": b.get("batch"),
|
||||
"stage": b.get("stage"),
|
||||
"path": b.get("path"),
|
||||
"suggested_batch_name": b.get("batch"),
|
||||
}
|
||||
)
|
||||
|
||||
return {"unmatched_count": len(unmatched), "items": unmatched[:100]}
|
||||
68
platform/as_platform/integrations/feishu_notify.py
Normal file
68
platform/as_platform/integrations/feishu_notify.py
Normal file
@@ -0,0 +1,68 @@
|
||||
"""飞书群消息通知(出站,内网可用)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from as_platform.auth.feishu import _get_tenant_access_token
|
||||
from as_platform.config import FEISHU_APP_ID, FEISHU_APP_SECRET, FEISHU_LABELING_CHAT_ID
|
||||
|
||||
IM_MSG_URL = "https://open.feishu.cn/open-apis/im/v1/messages"
|
||||
|
||||
|
||||
def is_notify_configured() -> bool:
|
||||
return bool(FEISHU_APP_ID and FEISHU_APP_SECRET and FEISHU_LABELING_CHAT_ID)
|
||||
|
||||
|
||||
def send_chat_text(text: str) -> dict[str, Any]:
|
||||
if not is_notify_configured():
|
||||
return {"ok": False, "message": "未配置 FEISHU_LABELING_CHAT_ID"}
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
token = _get_tenant_access_token(client)
|
||||
resp = client.post(
|
||||
IM_MSG_URL,
|
||||
params={"receive_id_type": "chat_id"},
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"receive_id": FEISHU_LABELING_CHAT_ID,
|
||||
"msg_type": "text",
|
||||
"content": json.dumps({"text": text}, ensure_ascii=False),
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("code") != 0:
|
||||
return {"ok": False, "message": data.get("msg") or "send failed"}
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
def send_chat_async(text: str) -> None:
|
||||
"""异步发送飞书消息,不阻塞主流程。"""
|
||||
import threading
|
||||
threading.Thread(target=_send_safe, args=(text,), daemon=True, name="feishu-notify").start()
|
||||
|
||||
|
||||
def _send_safe(text: str) -> None:
|
||||
try:
|
||||
send_chat_text(text)
|
||||
except Exception:
|
||||
pass # 通知失败不影响业务流程
|
||||
|
||||
|
||||
def notify_batch_progress(
|
||||
*,
|
||||
delivery_id: str,
|
||||
task: str,
|
||||
batch_name: str,
|
||||
progress: str,
|
||||
link: str,
|
||||
) -> dict[str, Any]:
|
||||
text = (
|
||||
f"[HSAP] 标注进度 {delivery_id or batch_name}\n"
|
||||
f"任务: {task} / 批次: {batch_name}\n"
|
||||
f"进度: {progress}\n"
|
||||
f"链接: {link}"
|
||||
)
|
||||
return send_chat_text(text)
|
||||
24
platform/as_platform/jobs/feishu_bitable_sync.py
Normal file
24
platform/as_platform/jobs/feishu_bitable_sync.py
Normal file
@@ -0,0 +1,24 @@
|
||||
"""飞书多维表格同步周期任务。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import FEISHU_BITABLE_AUTO_INGEST, FEISHU_BITABLE_SYNC_ENABLED
|
||||
from as_platform.integrations.feishu_bitable import is_bitable_configured
|
||||
from as_platform.integrations.feishu_bitable_ingest import process_pending_ingest
|
||||
from as_platform.integrations.feishu_bitable_sync import sync_hsap_to_bitable
|
||||
|
||||
|
||||
def run_sync_cycle() -> dict[str, Any]:
|
||||
if not is_bitable_configured():
|
||||
return {"ok": False, "message": "飞书多维表格未配置"}
|
||||
|
||||
out: dict[str, Any] = {"ok": True}
|
||||
if FEISHU_BITABLE_AUTO_INGEST:
|
||||
out["ingest"] = process_pending_ingest()
|
||||
out["sync"] = sync_hsap_to_bitable()
|
||||
return out
|
||||
|
||||
|
||||
def should_run_background_sync() -> bool:
|
||||
return FEISHU_BITABLE_SYNC_ENABLED and is_bitable_configured()
|
||||
@@ -62,12 +62,24 @@ def get_job(job_id: str) -> dict[str, Any] | None:
|
||||
return rec.to_dict() if rec else None
|
||||
|
||||
|
||||
def list_jobs(status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
|
||||
def list_jobs(
|
||||
status: str | None = None,
|
||||
*,
|
||||
offset: int = 0,
|
||||
limit: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
q = db.query(Job).order_by(Job.created_at.desc())
|
||||
if status:
|
||||
q = q.filter(Job.status == status)
|
||||
return [j.to_dict() for j in q.limit(limit).all()]
|
||||
total = q.count()
|
||||
rows = q.offset(max(0, offset)).limit(max(1, limit)).all()
|
||||
return {
|
||||
"items": [j.to_dict() for j in rows],
|
||||
"total": total,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
|
||||
def _patch(job_id: str, **fields: Any) -> dict[str, Any] | None:
|
||||
@@ -124,12 +136,25 @@ def _run_job(job_id: str) -> None:
|
||||
with trace_span("job_end", job_id=job_id, status="succeeded"):
|
||||
pass
|
||||
_sync_approval(job.get("approval_id"), "executed", persisted)
|
||||
if job.get("action") == "labeling_export":
|
||||
from as_platform.labeling.batch_stage import on_labeling_export_job_succeeded
|
||||
|
||||
on_labeling_export_job_succeeded(job)
|
||||
except Exception as e:
|
||||
_patch(job_id, status="failed", finished_at=_now(), result={"ok": False, "error": str(e)})
|
||||
publish("job.failed", {"job_id": job_id, "error": str(e)})
|
||||
with trace_span("job_end", job_id=job_id, status="failed", error=str(e)):
|
||||
pass
|
||||
_sync_approval(job.get("approval_id"), "failed", {"error": str(e)})
|
||||
if job.get("action") == "delivery_ingest":
|
||||
from as_platform.deliveries.service import mark_delivery_ingest_failed
|
||||
|
||||
params = job.get("params") or {}
|
||||
mark_delivery_ingest_failed(
|
||||
params.get("delivery_id"),
|
||||
job.get("approval_id"),
|
||||
str(e),
|
||||
)
|
||||
|
||||
|
||||
def _sync_approval(approval_id: str | None, status: str, result: dict) -> None:
|
||||
|
||||
@@ -19,6 +19,18 @@ AS_PY = ML_PY
|
||||
LONG_ACTIONS = {"train_dms", "train_lane", "pipeline_dms", "eval_dms", "eval_lane", "visualize_dms", "visualize_lane"}
|
||||
|
||||
|
||||
def _auto_snapshot(project: str, task: str = "") -> None:
|
||||
"""build 成功后自动创建数据集版本快照。"""
|
||||
try:
|
||||
from as_platform.data.versions import create_snapshot
|
||||
desc = f"自动快照 · build {project}"
|
||||
if task:
|
||||
desc += f"/{task}"
|
||||
create_snapshot(project, description=desc, author="system")
|
||||
except Exception:
|
||||
pass # 快照失败不影响 build
|
||||
|
||||
|
||||
def _run_ml(argv: list[str], timeout: int = 7200) -> dict[str, Any]:
|
||||
cmd = [sys.executable, str(ML_PY), *argv]
|
||||
proc = subprocess.run(cmd, cwd=str(WORKSPACE), capture_output=True, text=True, timeout=timeout)
|
||||
@@ -34,9 +46,14 @@ def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
track = p.get("track", "platform")
|
||||
if track == "local":
|
||||
from algorithms.dms_yolo.adapter import train_local
|
||||
return train_local(p["task"], p.get("mode", "full"), p.get("config_overrides"))
|
||||
return train_local(
|
||||
p["task"],
|
||||
p.get("mode", "full"),
|
||||
p.get("config_overrides"),
|
||||
submode=p.get("submode"),
|
||||
)
|
||||
from algorithms.dms_yolo.adapter import train_platform
|
||||
return train_platform(p["task"], p.get("mode", "full"))
|
||||
return train_platform(p["task"], p.get("mode", "full"), submode=p.get("submode"))
|
||||
|
||||
if action == "train_lane":
|
||||
track = p.get("track", "platform")
|
||||
@@ -69,10 +86,15 @@ def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
argv.append("--skip-validate")
|
||||
if p.get("no_refresh"):
|
||||
argv.append("--no-refresh")
|
||||
return _run_ml(argv)
|
||||
result = _run_ml(argv)
|
||||
# 自动创建数据集快照
|
||||
_auto_snapshot("dms", task=p.get("task", ""))
|
||||
return result
|
||||
|
||||
if action == "build_lane":
|
||||
return _run_ml(["build", "lane"])
|
||||
result = _run_ml(["build", "lane"])
|
||||
_auto_snapshot("lane")
|
||||
return result
|
||||
|
||||
if action == "enable_pack":
|
||||
return _run_ml(["enable", p["project"], p["pack"]])
|
||||
@@ -144,6 +166,20 @@ def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
)
|
||||
return {"ok": True, "stdout": "register_batch ok", "stderr": ""}
|
||||
|
||||
if action == "delivery_ingest":
|
||||
from as_platform.integrations.delivery_ingest import run_delivery_ingest
|
||||
|
||||
delivery_id = p.get("delivery_id") or ""
|
||||
if not delivery_id:
|
||||
raise ValueError("缺少 delivery_id")
|
||||
result = run_delivery_ingest(delivery_id)
|
||||
return {
|
||||
"ok": True,
|
||||
"stdout": json.dumps(result, ensure_ascii=False),
|
||||
"stderr": "",
|
||||
"result": result,
|
||||
}
|
||||
|
||||
if action == "analyze_uploaded_dataset":
|
||||
from as_platform.data.lake import analyze_uploaded_candidate
|
||||
|
||||
@@ -156,4 +192,76 @@ def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
"result": result,
|
||||
}
|
||||
|
||||
if action == "labeling_export":
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign
|
||||
from as_platform.labeling.annotate import resolve_campaign_batch_dir
|
||||
from as_platform.labeling.service import get_campaign
|
||||
|
||||
campaign_id = p.get("campaign_id", "")
|
||||
row = get_campaign(campaign_id)
|
||||
if not row:
|
||||
raise ValueError("campaign not found")
|
||||
task = row.get("task") or "dam"
|
||||
batch = row.get("batch") or ""
|
||||
pack = row.get("pack") or "dms_v2"
|
||||
export = row.get("export_default") or "yolo"
|
||||
if row.get("project") == "dms" and export in ("yolo", "yolo_pose") and batch:
|
||||
scripts_dir = WORKSPACE / "datasets" / "dms" / "scripts"
|
||||
if str(scripts_dir) not in sys.path:
|
||||
sys.path.insert(0, str(scripts_dir))
|
||||
from export_ls_to_yolo import export_batch
|
||||
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise ValueError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
export_mode = "pose" if export == "yolo_pose" else "detect"
|
||||
conv = export_batch(
|
||||
batch_dir,
|
||||
task,
|
||||
mode=export_mode,
|
||||
task_mode=row.get("mode"),
|
||||
)
|
||||
if conv.get("written", 0) == 0:
|
||||
raise ValueError(
|
||||
"export_ls_to_yolo: 无有效标注可导出 (written=0); "
|
||||
f"skipped_empty={conv.get('skipped_empty')} missing_ann={conv.get('missing_ann')}"
|
||||
)
|
||||
argv = ["build", "dms", task, "--pack", pack, "--batch", batch]
|
||||
result = _run_ml(argv)
|
||||
result["export_convert"] = conv
|
||||
return result
|
||||
if row.get("project") == "lane" and export == "lane_gt_txt":
|
||||
scripts_dir = WORKSPACE / "datasets" / "lane" / "scripts"
|
||||
if str(scripts_dir) not in sys.path:
|
||||
sys.path.insert(0, str(scripts_dir))
|
||||
from export_ls_to_lane_gt import export_batch
|
||||
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise ValueError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
conv = export_batch(batch_dir)
|
||||
if conv.get("written", 0) == 0:
|
||||
raise ValueError(
|
||||
"export_ls_to_lane_gt: 无有效标注可导出 (written=0); "
|
||||
f"skipped_empty={conv.get('skipped_empty')} missing_ann={conv.get('missing_ann')}"
|
||||
)
|
||||
argv = ["build", "lane"]
|
||||
result = _run_ml(argv)
|
||||
result["export_convert"] = conv
|
||||
return result
|
||||
return {
|
||||
"ok": True,
|
||||
"stdout": json.dumps({"export": export, "campaign": row}, ensure_ascii=False),
|
||||
"stderr": "",
|
||||
"message": f"export 类型 {export} 暂无 CLI,已记录",
|
||||
}
|
||||
|
||||
if action == "labeling_ml_predict":
|
||||
raise ValueError("labeling_ml_predict 已停用")
|
||||
|
||||
raise ValueError(f"未实现执行: {action}")
|
||||
|
||||
1
platform/as_platform/labeling/__init__.py
Normal file
1
platform/as_platform/labeling/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""标注 Campaign 与 scope 解析。"""
|
||||
287
platform/as_platform/labeling/annotate.py
Normal file
287
platform/as_platform/labeling/annotate.py
Normal file
@@ -0,0 +1,287 @@
|
||||
"""标注画布:批次目录、LS 配置 XML、任务列表、标注 JSON、媒体文件。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import yaml
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
from as_platform.data.batch import IMG_EXTS
|
||||
from as_platform.data.core import load_wf, proj_root, resolve_pack_dir
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign, User
|
||||
from as_platform.labeling.scope import (
|
||||
enrich_batch_labels,
|
||||
labeling_profile_key,
|
||||
load_dms_registry,
|
||||
load_labeling_registry,
|
||||
)
|
||||
|
||||
ANNOTATIONS_DIRNAME = "ls_annotations"
|
||||
|
||||
|
||||
def _label_config_dir(project: str) -> Path:
|
||||
return WORKSPACE / "datasets" / project / "configs" / "label_studio"
|
||||
|
||||
_FALLBACK_XML = """<View>
|
||||
<Image name="image" value="$image"/>
|
||||
<RectangleLabels name="label" toName="image">
|
||||
<Label value="object"/>
|
||||
</RectangleLabels>
|
||||
</View>"""
|
||||
|
||||
|
||||
def _load_campaign(campaign_id: str) -> LabelingCampaign | None:
|
||||
with session_scope() as db:
|
||||
return db.get(LabelingCampaign, campaign_id)
|
||||
|
||||
|
||||
def resolve_editor_xml(project: str, task: str, mode: str | None) -> str:
|
||||
reg = load_dms_registry() if project == "dms" else None
|
||||
pk = labeling_profile_key(project, task or "lane_v1", mode, reg)
|
||||
prof = (load_labeling_registry().get("profiles") or {}).get(pk) or {}
|
||||
default_tpl = "dam_15cls.xml" if project == "dms" else "lane_ufld_mask.xml"
|
||||
template = prof.get("editor_template") or default_tpl
|
||||
path = _label_config_dir(project) / template
|
||||
if path.is_file():
|
||||
return path.read_text(encoding="utf-8")
|
||||
return _FALLBACK_XML
|
||||
|
||||
|
||||
def resolve_campaign_batch_dir(camp: LabelingCampaign) -> Path:
|
||||
wf = load_wf()
|
||||
root = proj_root(wf, camp.project)
|
||||
if camp.project == "dms":
|
||||
reg = yaml.safe_load((root / wf["projects"]["dms"]["registry"]).read_text(encoding="utf-8"))
|
||||
tcfg = reg["tasks"][camp.task]
|
||||
if camp.location == "sources":
|
||||
if not camp.pack:
|
||||
raise ValueError("sources 批次需要 pack")
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, camp.pack)
|
||||
src_sub = (reg.get("ingest") or {}).get("sources_subdir", "sources")
|
||||
return (pack_dir / tcfg["task_dir"] / src_sub / camp.batch).resolve()
|
||||
if tcfg.get("type") == "multi" and camp.mode:
|
||||
from as_platform.labeling.scope import _dms_registry_api
|
||||
|
||||
get_mode_config, resolve_task_id, _ = _dms_registry_api()
|
||||
task_r, mode_r = resolve_task_id(camp.task, camp.mode)
|
||||
mcfg = get_mode_config(task_r, mode_r, reg)
|
||||
inbox_rel = mcfg.get("inbox")
|
||||
if inbox_rel:
|
||||
return (root / inbox_rel).resolve()
|
||||
mode = camp.mode
|
||||
if mode:
|
||||
return (root / "inbox" / camp.task / mode / camp.batch).resolve()
|
||||
return (root / "inbox" / camp.task / camp.batch).resolve()
|
||||
if camp.location == "pack" and camp.pack:
|
||||
try:
|
||||
from as_platform.data.core import resolve_pack
|
||||
|
||||
rel = resolve_pack("lane", root, wf, camp.pack)
|
||||
return (root / rel).resolve()
|
||||
except ValueError:
|
||||
return (root / camp.pack).resolve()
|
||||
return (root / "inbox" / camp.batch).resolve()
|
||||
|
||||
|
||||
def _iter_batch_images(batch_dir: Path) -> list[Path]:
|
||||
if not batch_dir.is_dir():
|
||||
return []
|
||||
candidates: list[Path] = []
|
||||
search_roots = [
|
||||
batch_dir / "images",
|
||||
batch_dir / "images" / "train",
|
||||
batch_dir,
|
||||
]
|
||||
seen: set[str] = set()
|
||||
for root in search_roots:
|
||||
if not root.is_dir():
|
||||
continue
|
||||
for p in sorted(root.rglob("*")):
|
||||
if not p.is_file() or p.suffix not in IMG_EXTS:
|
||||
continue
|
||||
key = str(p.resolve())
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
candidates.append(p.resolve())
|
||||
return candidates
|
||||
|
||||
|
||||
def _task_id_for_image(image_path: Path, batch_dir: Path) -> str:
|
||||
try:
|
||||
rel = image_path.relative_to(batch_dir)
|
||||
stem = rel.as_posix()
|
||||
except ValueError:
|
||||
stem = image_path.stem
|
||||
return hashlib.sha256(stem.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def _annotations_dir(batch_dir: Path) -> Path:
|
||||
d = batch_dir / "labels" / ANNOTATIONS_DIRNAME
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def sync_campaign_config_xml(camp: LabelingCampaign) -> str:
|
||||
"""始终与 labeling.registry + 模板文件对齐,避免 campaign 卡在旧 fallback XML。"""
|
||||
xml = resolve_editor_xml(camp.project, camp.task, camp.mode)
|
||||
if camp.config_xml != xml:
|
||||
camp.config_xml = xml
|
||||
return xml
|
||||
|
||||
|
||||
def campaign_bootstrap(campaign_id: str) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
sync_campaign_config_xml(camp)
|
||||
reg = load_dms_registry() if camp.project == "dms" else None
|
||||
row = enrich_batch_labels(camp.to_dict(), reg)
|
||||
row["config_xml"] = camp.config_xml
|
||||
try:
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
row["batch_path"] = str(batch_dir)
|
||||
row["image_count"] = len(_iter_batch_images(batch_dir))
|
||||
except Exception as e:
|
||||
row["batch_path"] = None
|
||||
row["image_count"] = 0
|
||||
row["batch_error"] = str(e)
|
||||
return row
|
||||
|
||||
|
||||
def campaign_tasks(
|
||||
campaign_id: str,
|
||||
*,
|
||||
offset: int = 0,
|
||||
limit: int = 50,
|
||||
user: User | None = None,
|
||||
assignee: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
images = _iter_batch_images(batch_dir)
|
||||
from as_platform.labeling.progress import get_assigned_task_ids, user_is_coordinator
|
||||
|
||||
filter_ids: set[str] | None = None
|
||||
if assignee == "me" and user:
|
||||
filter_ids = get_assigned_task_ids(campaign_id, user.id)
|
||||
if not filter_ids and not user_is_coordinator(user):
|
||||
return {
|
||||
"tasks": [],
|
||||
"total": 0,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"hint": "暂无分配给您的任务,请联系协调员在送标工作台均分任务",
|
||||
}
|
||||
|
||||
if filter_ids is not None:
|
||||
filtered = [img for img in images if _task_id_for_image(img, batch_dir) in filter_ids]
|
||||
images = filtered
|
||||
|
||||
total = len(images)
|
||||
slice_imgs = images[offset : offset + limit]
|
||||
tasks: list[dict[str, Any]] = []
|
||||
for img in slice_imgs:
|
||||
tid = _task_id_for_image(img, batch_dir)
|
||||
try:
|
||||
rel = img.relative_to(batch_dir).as_posix()
|
||||
except ValueError:
|
||||
rel = img.name
|
||||
media_path = quote(rel, safe="/")
|
||||
tasks.append(
|
||||
{
|
||||
"id": tid,
|
||||
"data": {
|
||||
"image": f"/api/v1/labeling/media/{campaign_id}/{media_path}",
|
||||
},
|
||||
"meta": {"filename": img.name, "relative_path": rel},
|
||||
}
|
||||
)
|
||||
out: dict[str, Any] = {"tasks": tasks, "total": total, "offset": offset, "limit": limit}
|
||||
if filter_ids is not None and user and assignee == "me":
|
||||
out["my_assigned"] = len(filter_ids)
|
||||
return out
|
||||
|
||||
|
||||
def resolve_media_file(campaign_id: str, rel_path: str) -> Path:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
clean = Path(rel_path)
|
||||
if clean.is_absolute() or ".." in clean.parts:
|
||||
raise PermissionError("invalid path")
|
||||
target = (batch_dir / clean).resolve()
|
||||
if not target.is_file() or not target.is_relative_to(batch_dir.resolve()):
|
||||
raise FileNotFoundError("media not found")
|
||||
return target
|
||||
|
||||
|
||||
def get_annotation(campaign_id: str, task_id: str) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
path = _annotations_dir(batch_dir) / f"{task_id}.json"
|
||||
if not path.is_file():
|
||||
return {"task_id": task_id, "result": None, "annotations": []}
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
return data
|
||||
|
||||
|
||||
def save_annotation(
|
||||
campaign_id: str,
|
||||
task_id: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
user: User | None = None,
|
||||
) -> dict[str, Any]:
|
||||
from as_platform.labeling.progress import assert_can_save_task, mark_task_completed
|
||||
|
||||
if user:
|
||||
assert_can_save_task(campaign_id, task_id, user)
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
path = _annotations_dir(batch_dir) / f"{task_id}.json"
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
extra: dict[str, Any] = {}
|
||||
if user:
|
||||
extra["completed_by_user_id"] = user.id
|
||||
extra["completed_at"] = now
|
||||
out = {"task_id": task_id, **payload, **extra}
|
||||
path.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
if user and _annotation_has_result(path):
|
||||
mark_task_completed(campaign_id, task_id, user.id)
|
||||
return {"ok": True, "path": str(path)}
|
||||
|
||||
|
||||
def _annotation_has_result(path: Path) -> bool:
|
||||
if not path.is_file():
|
||||
return False
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return False
|
||||
result = data.get("result")
|
||||
if result is None:
|
||||
return False
|
||||
if isinstance(result, list):
|
||||
return len(result) > 0
|
||||
if isinstance(result, dict):
|
||||
return len(result) > 0
|
||||
return bool(result)
|
||||
81
platform/as_platform/labeling/batch_stage.py
Normal file
81
platform/as_platform/labeling/batch_stage.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""同步 inbox/sources 批次 batch.meta.yaml 的 stage,与 Campaign 状态一致。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from as_platform.data.batch import read_meta, write_meta
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign
|
||||
from as_platform.labeling.annotate import resolve_campaign_batch_dir
|
||||
|
||||
|
||||
def batch_has_yolo_labels(batch_dir: Path) -> bool:
|
||||
"""批次是否已有导出的 YOLO txt(labels/train 或 labels 根目录)。"""
|
||||
for sub in ("labels/train", "labels"):
|
||||
d = batch_dir / sub
|
||||
if d.is_dir() and any(d.glob("*.txt")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def batch_has_lane_labels(batch_dir: Path) -> bool:
|
||||
"""批次是否已有 UFLD mask 清单(list/train_gt.txt + annotations/*.png)。"""
|
||||
list_path = batch_dir / "list" / "train_gt.txt"
|
||||
if not list_path.is_file():
|
||||
return False
|
||||
ann_dir = batch_dir / "annotations"
|
||||
if not ann_dir.is_dir():
|
||||
return False
|
||||
return any(ann_dir.rglob("*.png"))
|
||||
|
||||
|
||||
def update_campaign_batch_meta_stage(camp: LabelingCampaign, stage: str) -> bool:
|
||||
try:
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
except Exception:
|
||||
return False
|
||||
if not batch_dir.is_dir():
|
||||
return False
|
||||
meta = read_meta(batch_dir) or {}
|
||||
meta["stage"] = stage
|
||||
meta.setdefault("project", camp.project)
|
||||
meta.setdefault("task", camp.task)
|
||||
meta.setdefault("batch", camp.batch)
|
||||
meta.setdefault("location", camp.location or "inbox")
|
||||
if camp.mode:
|
||||
meta.setdefault("mode", camp.mode)
|
||||
write_meta(batch_dir, meta)
|
||||
return True
|
||||
|
||||
|
||||
def update_campaign_batch_meta_stage_by_id(campaign_id: str, stage: str) -> bool:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
return False
|
||||
return update_campaign_batch_meta_stage(camp, stage)
|
||||
|
||||
|
||||
def on_labeling_export_job_succeeded(job: dict) -> None:
|
||||
"""导出 Job 成功且批次已有训练标签时进入 returned(待入库)。"""
|
||||
if job.get("action") != "labeling_export":
|
||||
return
|
||||
params = job.get("params") or {}
|
||||
cid = params.get("campaign_id")
|
||||
if not cid:
|
||||
return
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, str(cid))
|
||||
if not camp:
|
||||
return
|
||||
try:
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
except Exception:
|
||||
return
|
||||
has_labels = (
|
||||
batch_has_lane_labels(batch_dir)
|
||||
if camp.project == "lane"
|
||||
else batch_has_yolo_labels(batch_dir)
|
||||
)
|
||||
if has_labels:
|
||||
update_campaign_batch_meta_stage_by_id(str(cid), "returned")
|
||||
94
platform/as_platform/labeling/lock.py
Normal file
94
platform/as_platform/labeling/lock.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""标注任务 Redis 互斥锁(campaign + task 粒度)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from as_platform.redis.bus import get_redis
|
||||
|
||||
LOCK_TTL_SEC = 300
|
||||
_LOCK_PREFIX = "labeling:lock:"
|
||||
# API 进程内回退(无 Redis 时)
|
||||
_memory: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
def _key(campaign_id: str, task_id: str) -> str:
|
||||
return f"{_LOCK_PREFIX}{campaign_id}:{task_id}"
|
||||
|
||||
|
||||
def _parse_holder(raw: str | None) -> dict[str, Any] | None:
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return {"name": raw}
|
||||
|
||||
|
||||
def acquire_lock(campaign_id: str, task_id: str, *, user_id: int, user_name: str) -> dict[str, Any]:
|
||||
payload = json.dumps({"user_id": user_id, "name": user_name}, ensure_ascii=False)
|
||||
r = get_redis()
|
||||
if not r:
|
||||
now = time.time()
|
||||
mem = _memory.get(_key(campaign_id, task_id))
|
||||
if mem and mem.get("user_id") != user_id and now < mem.get("expires_at", 0):
|
||||
return {"ok": False, "holder": mem.get("name"), "user_id": mem.get("user_id")}
|
||||
_memory[_key(campaign_id, task_id)] = {
|
||||
"user_id": user_id,
|
||||
"name": user_name,
|
||||
"expires_at": now + LOCK_TTL_SEC,
|
||||
}
|
||||
return {"ok": True, "holder": user_name, "ttl_sec": LOCK_TTL_SEC, "backend": "memory"}
|
||||
|
||||
key = _key(campaign_id, task_id)
|
||||
if r.set(key, payload, nx=True, ex=LOCK_TTL_SEC):
|
||||
return {"ok": True, "holder": user_name, "ttl_sec": LOCK_TTL_SEC, "backend": "redis"}
|
||||
existing = _parse_holder(r.get(key))
|
||||
if existing and existing.get("user_id") == user_id:
|
||||
r.expire(key, LOCK_TTL_SEC)
|
||||
return {"ok": True, "holder": user_name, "ttl_sec": LOCK_TTL_SEC, "renewed": True, "backend": "redis"}
|
||||
return {
|
||||
"ok": False,
|
||||
"holder": (existing or {}).get("name"),
|
||||
"user_id": (existing or {}).get("user_id"),
|
||||
"backend": "redis",
|
||||
}
|
||||
|
||||
|
||||
def release_lock(campaign_id: str, task_id: str, *, user_id: int) -> dict[str, Any]:
|
||||
r = get_redis()
|
||||
if not r:
|
||||
key = _key(campaign_id, task_id)
|
||||
mem = _memory.get(key)
|
||||
if mem and mem.get("user_id") == user_id:
|
||||
_memory.pop(key, None)
|
||||
return {"ok": True, "released": True, "backend": "memory"}
|
||||
return {"ok": True, "released": False, "backend": "memory"}
|
||||
|
||||
key = _key(campaign_id, task_id)
|
||||
existing = _parse_holder(r.get(key))
|
||||
if not existing:
|
||||
return {"ok": True, "released": False, "backend": "redis"}
|
||||
if existing.get("user_id") != user_id:
|
||||
return {"ok": False, "holder": existing.get("name"), "backend": "redis"}
|
||||
r.delete(key)
|
||||
return {"ok": True, "released": True, "backend": "redis"}
|
||||
|
||||
|
||||
def renew_lock(campaign_id: str, task_id: str, *, user_id: int) -> dict[str, Any]:
|
||||
r = get_redis()
|
||||
if not r:
|
||||
key = _key(campaign_id, task_id)
|
||||
mem = _memory.get(key)
|
||||
if mem and mem.get("user_id") == user_id:
|
||||
mem["expires_at"] = time.time() + LOCK_TTL_SEC
|
||||
return {"ok": True, "ttl_sec": LOCK_TTL_SEC, "backend": "memory"}
|
||||
return {"ok": False, "backend": "memory"}
|
||||
|
||||
key = _key(campaign_id, task_id)
|
||||
existing = _parse_holder(r.get(key))
|
||||
if not existing or existing.get("user_id") != user_id:
|
||||
return {"ok": False, "holder": (existing or {}).get("name"), "backend": "redis"}
|
||||
r.expire(key, LOCK_TTL_SEC)
|
||||
return {"ok": True, "ttl_sec": LOCK_TTL_SEC, "backend": "redis"}
|
||||
313
platform/as_platform/labeling/progress.py
Normal file
313
platform/as_platform/labeling/progress.py
Normal file
@@ -0,0 +1,313 @@
|
||||
"""标注进度统计与按人分包。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign, LabelingTaskAssignment, User
|
||||
from as_platform.labeling.annotate import (
|
||||
_annotations_dir,
|
||||
_iter_batch_images,
|
||||
_task_id_for_image,
|
||||
resolve_campaign_batch_dir,
|
||||
)
|
||||
|
||||
COORDINATOR_ROLES = frozenset({"labeler", "admin", "engineer", "reviewer"})
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def user_role_codes(user: User) -> set[str]:
|
||||
return {r.code for r in (user.roles or [])}
|
||||
|
||||
|
||||
def user_is_coordinator(user: User) -> bool:
|
||||
codes = user_role_codes(user)
|
||||
if codes & COORDINATOR_ROLES:
|
||||
return True
|
||||
perms: set[str] = set()
|
||||
for r in user.roles or []:
|
||||
for p in r.permissions or []:
|
||||
if p.code:
|
||||
perms.add(p.code)
|
||||
return "*" in perms or "write:labeling_assign" in perms
|
||||
|
||||
|
||||
def list_campaign_task_ids(campaign_id: str) -> list[str]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
images = _iter_batch_images(batch_dir)
|
||||
return [_task_id_for_image(img, batch_dir) for img in images]
|
||||
|
||||
|
||||
def _annotation_has_result(path) -> bool:
|
||||
if not path.is_file():
|
||||
return False
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return False
|
||||
result = data.get("result")
|
||||
if result is None:
|
||||
return False
|
||||
if isinstance(result, list):
|
||||
return len(result) > 0
|
||||
if isinstance(result, dict):
|
||||
return len(result) > 0
|
||||
return bool(result)
|
||||
|
||||
|
||||
def count_completed_tasks(batch_dir) -> set[str]:
|
||||
ann_dir = _annotations_dir(batch_dir)
|
||||
done: set[str] = set()
|
||||
if not ann_dir.is_dir():
|
||||
return done
|
||||
for p in ann_dir.glob("*.json"):
|
||||
if _annotation_has_result(p):
|
||||
done.add(p.stem)
|
||||
return done
|
||||
|
||||
|
||||
def _user_name_map(db, user_ids: set[int]) -> dict[int, str]:
|
||||
if not user_ids:
|
||||
return {}
|
||||
rows = db.query(User).filter(User.id.in_(user_ids)).all()
|
||||
return {u.id: u.name or f"user-{u.id}" for u in rows}
|
||||
|
||||
|
||||
def campaign_progress(campaign_id: str) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
all_ids = [_task_id_for_image(img, batch_dir) for img in _iter_batch_images(batch_dir)]
|
||||
completed_ids = count_completed_tasks(batch_dir)
|
||||
rows = (
|
||||
db.query(LabelingTaskAssignment)
|
||||
.filter(LabelingTaskAssignment.campaign_id == campaign_id)
|
||||
.all()
|
||||
)
|
||||
user_ids = {r.user_id for r in rows} | {r.completed_by_user_id for r in rows if r.completed_by_user_id}
|
||||
names = _user_name_map(db, user_ids)
|
||||
|
||||
total = len(all_ids)
|
||||
completed = len(completed_ids)
|
||||
assigned = len(rows)
|
||||
by_user_agg: dict[int, dict[str, int]] = defaultdict(lambda: {"assigned": 0, "completed": 0})
|
||||
for r in rows:
|
||||
by_user_agg[r.user_id]["assigned"] += 1
|
||||
if r.completed_at or r.task_id in completed_ids:
|
||||
by_user_agg[r.user_id]["completed"] += 1
|
||||
|
||||
by_user = []
|
||||
for uid, stats in sorted(by_user_agg.items(), key=lambda x: names.get(x[0], "")):
|
||||
a = stats["assigned"]
|
||||
c = stats["completed"]
|
||||
by_user.append(
|
||||
{
|
||||
"user_id": uid,
|
||||
"name": names.get(uid, f"user-{uid}"),
|
||||
"assigned": a,
|
||||
"completed": c,
|
||||
"percent": round(100.0 * c / a, 1) if a else 0.0,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"campaign_id": campaign_id,
|
||||
"total_tasks": total,
|
||||
"completed_tasks": completed,
|
||||
"assigned_tasks": assigned,
|
||||
"unassigned_tasks": max(0, total - assigned),
|
||||
"percent": round(100.0 * completed / total, 1) if total else 0.0,
|
||||
"by_user": by_user,
|
||||
}
|
||||
|
||||
|
||||
def campaign_progress_summary(campaign_id: str) -> dict[str, int]:
|
||||
try:
|
||||
p = campaign_progress(campaign_id)
|
||||
return {
|
||||
"total_tasks": p["total_tasks"],
|
||||
"completed_tasks": p["completed_tasks"],
|
||||
"assigned_tasks": p["assigned_tasks"],
|
||||
}
|
||||
except FileNotFoundError:
|
||||
return {"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0}
|
||||
|
||||
|
||||
def get_assigned_task_ids(campaign_id: str, user_id: int | None = None) -> set[str]:
|
||||
with session_scope() as db:
|
||||
q = db.query(LabelingTaskAssignment.task_id).filter(
|
||||
LabelingTaskAssignment.campaign_id == campaign_id
|
||||
)
|
||||
if user_id is not None:
|
||||
q = q.filter(LabelingTaskAssignment.user_id == user_id)
|
||||
return {row[0] for row in q.all()}
|
||||
|
||||
|
||||
def _assign_result(campaign_id: str, created: int) -> dict[str, Any]:
|
||||
prog = campaign_progress(campaign_id)
|
||||
return {"assigned": created, "by_user": prog["by_user"], "progress": prog}
|
||||
|
||||
|
||||
def assign_tasks_even(
|
||||
campaign_id: str,
|
||||
user_ids: list[int],
|
||||
*,
|
||||
assigned_by_user_id: int,
|
||||
) -> dict[str, Any]:
|
||||
if not user_ids:
|
||||
raise ValueError("user_ids 不能为空")
|
||||
now = _utcnow()
|
||||
all_ids = list_campaign_task_ids(campaign_id)
|
||||
with session_scope() as db:
|
||||
existing = {
|
||||
r.task_id
|
||||
for r in db.query(LabelingTaskAssignment)
|
||||
.filter(LabelingTaskAssignment.campaign_id == campaign_id)
|
||||
.all()
|
||||
}
|
||||
unassigned = [tid for tid in all_ids if tid not in existing]
|
||||
users = db.query(User).filter(User.id.in_(user_ids)).all()
|
||||
if len(users) != len(set(user_ids)):
|
||||
raise ValueError("存在无效 user_id")
|
||||
created = 0
|
||||
for i, tid in enumerate(unassigned):
|
||||
uid = user_ids[i % len(user_ids)]
|
||||
db.add(
|
||||
LabelingTaskAssignment(
|
||||
campaign_id=campaign_id,
|
||||
task_id=tid,
|
||||
user_id=uid,
|
||||
assigned_by_user_id=assigned_by_user_id,
|
||||
assigned_at=now,
|
||||
)
|
||||
)
|
||||
created += 1
|
||||
db.flush()
|
||||
return _assign_result(campaign_id, created)
|
||||
|
||||
|
||||
def assign_tasks_explicit(
|
||||
campaign_id: str,
|
||||
items: list[dict[str, Any]],
|
||||
*,
|
||||
assigned_by_user_id: int,
|
||||
) -> dict[str, Any]:
|
||||
now = _utcnow()
|
||||
all_set = set(list_campaign_task_ids(campaign_id))
|
||||
created = 0
|
||||
with session_scope() as db:
|
||||
existing = {
|
||||
r.task_id
|
||||
for r in db.query(LabelingTaskAssignment)
|
||||
.filter(LabelingTaskAssignment.campaign_id == campaign_id)
|
||||
.all()
|
||||
}
|
||||
for item in items:
|
||||
uid = int(item["user_id"])
|
||||
user = db.get(User, uid)
|
||||
if not user:
|
||||
raise ValueError(f"用户不存在: {uid}")
|
||||
for tid in item.get("task_ids") or []:
|
||||
if tid not in all_set:
|
||||
raise ValueError(f"无效 task_id: {tid}")
|
||||
if tid in existing:
|
||||
continue
|
||||
db.add(
|
||||
LabelingTaskAssignment(
|
||||
campaign_id=campaign_id,
|
||||
task_id=tid,
|
||||
user_id=uid,
|
||||
assigned_by_user_id=assigned_by_user_id,
|
||||
assigned_at=now,
|
||||
)
|
||||
)
|
||||
existing.add(tid)
|
||||
created += 1
|
||||
db.flush()
|
||||
return _assign_result(campaign_id, created)
|
||||
|
||||
|
||||
def reassign_task(campaign_id: str, task_id: str, user_id: int) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
row = (
|
||||
db.query(LabelingTaskAssignment)
|
||||
.filter(
|
||||
LabelingTaskAssignment.campaign_id == campaign_id,
|
||||
LabelingTaskAssignment.task_id == task_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not row:
|
||||
raise FileNotFoundError("assignment not found")
|
||||
user = db.get(User, user_id)
|
||||
if not user:
|
||||
raise ValueError(f"用户不存在: {user_id}")
|
||||
row.user_id = user_id
|
||||
db.flush()
|
||||
return row.to_dict()
|
||||
|
||||
|
||||
def release_task_assignment(campaign_id: str, task_id: str) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
row = (
|
||||
db.query(LabelingTaskAssignment)
|
||||
.filter(
|
||||
LabelingTaskAssignment.campaign_id == campaign_id,
|
||||
LabelingTaskAssignment.task_id == task_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not row:
|
||||
raise FileNotFoundError("assignment not found")
|
||||
db.delete(row)
|
||||
db.flush()
|
||||
return {"ok": True, "released": task_id}
|
||||
|
||||
|
||||
def assert_can_save_task(campaign_id: str, task_id: str, user: User) -> None:
|
||||
if user_is_coordinator(user):
|
||||
return
|
||||
codes = user_role_codes(user)
|
||||
if "vendor_labeler" in codes:
|
||||
return
|
||||
with session_scope() as db:
|
||||
row = (
|
||||
db.query(LabelingTaskAssignment)
|
||||
.filter(
|
||||
LabelingTaskAssignment.campaign_id == campaign_id,
|
||||
LabelingTaskAssignment.task_id == task_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not row:
|
||||
raise PermissionError("该图未分配给您,请联系协调员分包")
|
||||
if row.user_id != user.id:
|
||||
raise PermissionError("该图已分配给其他标注员")
|
||||
|
||||
|
||||
def mark_task_completed(campaign_id: str, task_id: str, user_id: int) -> None:
|
||||
now = _utcnow()
|
||||
with session_scope() as db:
|
||||
row = (
|
||||
db.query(LabelingTaskAssignment)
|
||||
.filter(
|
||||
LabelingTaskAssignment.campaign_id == campaign_id,
|
||||
LabelingTaskAssignment.task_id == task_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if row and not row.completed_at:
|
||||
row.completed_at = now
|
||||
row.completed_by_user_id = user_id
|
||||
86
platform/as_platform/labeling/scope.py
Normal file
86
platform/as_platform/labeling/scope.py
Normal file
@@ -0,0 +1,86 @@
|
||||
"""CatalogScope 与 registry 对齐。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
|
||||
DOMAIN_LABELS = {"dms": "舱内 DMS", "forward": "前向 ADAS"}
|
||||
|
||||
|
||||
def format_scope_key(project: str, task: str, mode: str | None = None) -> str:
|
||||
if project == "lane":
|
||||
return f"lane:{task}"
|
||||
if mode:
|
||||
return f"dms:{task}:{mode}"
|
||||
return f"dms:{task}"
|
||||
|
||||
|
||||
def _dms_registry_api():
|
||||
import sys
|
||||
from pathlib import Path
|
||||
p = WORKSPACE / "datasets" / "dms" / "scripts"
|
||||
if str(p) not in sys.path:
|
||||
sys.path.insert(0, str(p))
|
||||
from task_registry import get_mode_config, resolve_task_id, train_yaml_key
|
||||
return get_mode_config, resolve_task_id, train_yaml_key
|
||||
|
||||
|
||||
def load_dms_registry() -> dict:
|
||||
path = WORKSPACE / "datasets" / "dms" / "datasets.registry.yaml"
|
||||
return yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def load_labeling_registry() -> dict[str, Any]:
|
||||
path = WORKSPACE / "datasets" / "labeling.registry.yaml"
|
||||
if not path.is_file():
|
||||
return {"profiles": {}}
|
||||
return yaml.safe_load(path.read_text(encoding="utf-8")) or {"profiles": {}}
|
||||
|
||||
|
||||
def labeling_profile_key(project: str, task: str, mode: str | None, reg: dict | None = None) -> str:
|
||||
if project == "lane":
|
||||
return f"lane__{task}"
|
||||
get_mode_config, resolve_task_id, train_yaml_key = _dms_registry_api()
|
||||
reg = reg or load_dms_registry()
|
||||
task, mode = resolve_task_id(task, mode)
|
||||
return train_yaml_key(task, mode, reg)
|
||||
|
||||
|
||||
def enrich_batch_labels(batch: dict[str, Any], reg: dict | None = None) -> dict[str, Any]:
|
||||
project = batch.get("project") or "dms"
|
||||
task = batch.get("task") or ""
|
||||
mode = batch.get("mode")
|
||||
out = dict(batch)
|
||||
out["scope_key"] = format_scope_key(project, task, mode)
|
||||
if project == "dms":
|
||||
reg = reg or load_dms_registry()
|
||||
get_mode_config, resolve_task_id, _ = _dms_registry_api()
|
||||
try:
|
||||
task_r, mode_r = resolve_task_id(task, mode)
|
||||
mcfg = get_mode_config(task_r, mode_r, reg)
|
||||
domain = mcfg.get("domain") or "dms"
|
||||
out["domain"] = domain
|
||||
out["domain_label"] = DOMAIN_LABELS.get(domain, domain)
|
||||
out["task_label"] = mcfg.get("label") or task
|
||||
if mode_r:
|
||||
modes = (reg.get("tasks") or {}).get(task_r, {}).get("modes") or {}
|
||||
out["mode_label"] = (modes.get(mode_r) or {}).get("label") or mode_r
|
||||
except Exception:
|
||||
out["domain"] = "dms"
|
||||
out["domain_label"] = DOMAIN_LABELS["dms"]
|
||||
else:
|
||||
out["domain_label"] = "车道线 Lane"
|
||||
out["task_label"] = task
|
||||
try:
|
||||
pk = labeling_profile_key(project, task, mode, reg if project == "dms" else None)
|
||||
out["labeling_profile"] = pk
|
||||
prof = (load_labeling_registry().get("profiles") or {}).get(pk)
|
||||
if prof:
|
||||
out["export_default"] = prof.get("export_default")
|
||||
out["ml_adapter"] = prof.get("ml_adapter")
|
||||
except Exception:
|
||||
out["labeling_profile"] = None
|
||||
return out
|
||||
401
platform/as_platform/labeling/service.py
Normal file
401
platform/as_platform/labeling/service.py
Normal file
@@ -0,0 +1,401 @@
|
||||
"""Campaign 与 pending 批次合并列表。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
from as_platform.data.core import get_pending_report, load_wf
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign, LabelingExportJob, User
|
||||
from as_platform.jobs.queue import enqueue_job, get_job
|
||||
from as_platform.labeling.annotate import resolve_editor_xml, sync_campaign_config_xml
|
||||
from as_platform.labeling.batch_stage import (
|
||||
on_labeling_export_job_succeeded,
|
||||
update_campaign_batch_meta_stage,
|
||||
)
|
||||
from as_platform.labeling.scope import (
|
||||
enrich_batch_labels,
|
||||
format_scope_key,
|
||||
load_dms_registry,
|
||||
load_labeling_registry,
|
||||
)
|
||||
|
||||
|
||||
def _campaign_id(project: str, task: str, mode: str | None, batch: str, location: str) -> str:
|
||||
sk = format_scope_key(project, task, mode)
|
||||
raw = f"{sk}:{batch}:{location}"
|
||||
return hashlib.sha256(raw.encode()).hexdigest()[:20]
|
||||
|
||||
|
||||
def _parse_scope_key(scope_key: str) -> tuple[str, str, str | None]:
|
||||
parts = scope_key.split(":")
|
||||
if parts[0] == "lane":
|
||||
return "lane", parts[1] if len(parts) > 1 else "lane_v1", None
|
||||
if len(parts) >= 3:
|
||||
return "dms", parts[1], parts[2]
|
||||
if len(parts) == 2:
|
||||
return "dms", parts[1], None
|
||||
return "dms", parts[-1], None
|
||||
|
||||
|
||||
def _registry_fallback_batches(wf: dict, reg: dict) -> list[dict[str, Any]]:
|
||||
"""labeling.registry 中有配置但 pending 未扫到的批次(如空 inbox)。"""
|
||||
from pathlib import Path
|
||||
|
||||
from as_platform.data.batch import enrich_batch
|
||||
from as_platform.data.core import proj_root
|
||||
|
||||
profiles = load_labeling_registry().get("profiles") or {}
|
||||
rows: list[dict[str, Any]] = []
|
||||
dms_root = proj_root(wf, "dms")
|
||||
for _pk, prof in profiles.items():
|
||||
scope_key = prof.get("scope_key") or ""
|
||||
project, task, mode = _parse_scope_key(scope_key)
|
||||
if project != "dms":
|
||||
continue
|
||||
batch = mode or task
|
||||
batch_dir = None
|
||||
if mode:
|
||||
try:
|
||||
import sys
|
||||
|
||||
scripts = WORKSPACE / "datasets" / "dms" / "scripts"
|
||||
if str(scripts) not in sys.path:
|
||||
sys.path.insert(0, str(scripts))
|
||||
from task_registry import inbox_dir, resolve_task_id
|
||||
|
||||
task_r, mode_r = resolve_task_id(task, mode)
|
||||
batch_dir = inbox_dir(dms_root, task_r, mode_r, reg)
|
||||
except Exception:
|
||||
batch_dir = dms_root / "inbox" / task / mode
|
||||
else:
|
||||
batch_dir = dms_root / "inbox" / task / batch
|
||||
if not isinstance(batch_dir, Path) or not batch_dir.is_dir():
|
||||
row = {
|
||||
"project": project,
|
||||
"task": task,
|
||||
"mode": mode,
|
||||
"batch": batch,
|
||||
"stage": "raw_pool",
|
||||
"location": "inbox",
|
||||
"path": str(batch_dir) if batch_dir else "",
|
||||
"counts": {"images": 0, "labels": 0},
|
||||
"registry_only": True,
|
||||
}
|
||||
else:
|
||||
row = enrich_batch(
|
||||
batch_dir,
|
||||
project=project,
|
||||
task=task,
|
||||
pack=None,
|
||||
batch=batch,
|
||||
location="inbox",
|
||||
)
|
||||
row["mode"] = mode
|
||||
row["scope_key"] = scope_key
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def list_labeling_batches(
|
||||
*,
|
||||
stage: str | None = None,
|
||||
offset: int = 0,
|
||||
limit: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
wf = load_wf()
|
||||
report = get_pending_report(wf)
|
||||
reg = load_dms_registry()
|
||||
items: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
allowed_stages = ("raw_pool", "out_for_labeling", "returned", "labeling_submitted", "in_review", "review_approved", "review_rejected")
|
||||
|
||||
def _append(b: dict[str, Any]) -> None:
|
||||
if stage and b.get("stage") != stage:
|
||||
return
|
||||
if b.get("stage") not in allowed_stages:
|
||||
return
|
||||
row = enrich_batch_labels(b, reg)
|
||||
cid = _campaign_id(
|
||||
row["project"], row.get("task") or "", row.get("mode"), row["batch"], row.get("location") or "inbox"
|
||||
)
|
||||
key = f"{cid}"
|
||||
if key in seen:
|
||||
return
|
||||
seen.add(key)
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, cid)
|
||||
status = camp.status if camp else "not_opened"
|
||||
if camp:
|
||||
row["assigned_to_user_id"] = camp.assigned_to_user_id
|
||||
row["assigned_to_name"] = camp.assigned_to_name
|
||||
row["campaign_id"] = cid
|
||||
row["campaign_status"] = status
|
||||
if camp and status in ("in_progress", "labeling_submitted"):
|
||||
try:
|
||||
from as_platform.labeling.progress import campaign_progress_summary
|
||||
|
||||
row.update(campaign_progress_summary(cid))
|
||||
except Exception:
|
||||
row.update({"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0})
|
||||
items.append(row)
|
||||
|
||||
for b in report.get("batches", []):
|
||||
_append(b)
|
||||
|
||||
for b in _registry_fallback_batches(wf, reg):
|
||||
_append(b)
|
||||
|
||||
total = len(items)
|
||||
page = items[max(0, offset) : max(0, offset) + max(1, limit)]
|
||||
return {
|
||||
"items": page,
|
||||
"total": total,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"updated_at": report.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
def open_campaign(
|
||||
*,
|
||||
project: str,
|
||||
task: str,
|
||||
batch: str,
|
||||
mode: str | None = None,
|
||||
pack: str | None = None,
|
||||
location: str = "inbox",
|
||||
) -> dict[str, Any]:
|
||||
cid = _campaign_id(project, task, mode, batch, location)
|
||||
config_xml = resolve_editor_xml(project, task, mode)
|
||||
now = datetime.now(timezone.utc)
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, cid)
|
||||
if not camp:
|
||||
camp = LabelingCampaign(
|
||||
id=cid,
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
batch=batch,
|
||||
pack=pack,
|
||||
location=location,
|
||||
status="in_progress",
|
||||
config_xml=config_xml,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
db.add(camp)
|
||||
else:
|
||||
camp.status = "in_progress"
|
||||
camp.updated_at = now
|
||||
sync_campaign_config_xml(camp)
|
||||
db.flush()
|
||||
out = camp.to_dict()
|
||||
out["config_xml"] = camp.config_xml
|
||||
update_campaign_batch_meta_stage(camp, "out_for_labeling")
|
||||
reg = load_dms_registry() if project == "dms" else None
|
||||
row = enrich_batch_labels(out, reg)
|
||||
row["stage"] = "out_for_labeling"
|
||||
return row
|
||||
|
||||
|
||||
def get_campaign(campaign_id: str) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
return None
|
||||
row = camp.to_dict()
|
||||
row["config_xml"] = camp.config_xml
|
||||
reg = load_dms_registry() if row.get("project") == "dms" else None
|
||||
return enrich_batch_labels(row, reg)
|
||||
|
||||
|
||||
def _export_job_id() -> str:
|
||||
return f"lej-{uuid.uuid4().hex[:16]}"
|
||||
|
||||
|
||||
def _record_export_job(campaign_id: str, action: str, job: dict[str, Any]) -> dict[str, Any]:
|
||||
ej_id = _export_job_id()
|
||||
job_id = job.get("id")
|
||||
now = datetime.now(timezone.utc)
|
||||
with session_scope() as db:
|
||||
ej = LabelingExportJob(
|
||||
id=ej_id,
|
||||
campaign_id=campaign_id,
|
||||
action=action,
|
||||
job_id=job_id,
|
||||
status=job.get("status") or "queued",
|
||||
created_at=now,
|
||||
)
|
||||
db.add(ej)
|
||||
out = get_export_job(ej_id)
|
||||
return out or {"id": ej_id, "campaign_id": campaign_id, "action": action, "job_id": job_id}
|
||||
|
||||
|
||||
def _sync_export_job_from_queue(ej: LabelingExportJob) -> None:
|
||||
if not ej.job_id:
|
||||
return
|
||||
job = get_job(ej.job_id)
|
||||
if not job:
|
||||
return
|
||||
ej.status = job.get("status") or ej.status
|
||||
if job.get("finished_at"):
|
||||
try:
|
||||
ej.finished_at = datetime.fromisoformat(str(job["finished_at"]).replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
pass
|
||||
if job.get("result") is not None:
|
||||
ej.result_json = json.dumps(job.get("result"), ensure_ascii=False)
|
||||
if ej.action == "labeling_export" and ej.status in ("succeeded", "completed"):
|
||||
on_labeling_export_job_succeeded(
|
||||
{"action": "labeling_export", "params": {"campaign_id": ej.campaign_id}}
|
||||
)
|
||||
|
||||
|
||||
def get_export_job(export_job_id: str) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
ej = db.get(LabelingExportJob, export_job_id)
|
||||
if not ej:
|
||||
return None
|
||||
_sync_export_job_from_queue(ej)
|
||||
db.flush()
|
||||
return ej.to_dict()
|
||||
|
||||
|
||||
def list_campaign_export_jobs(campaign_id: str, *, limit: int = 30) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
rows = (
|
||||
db.query(LabelingExportJob)
|
||||
.filter_by(campaign_id=campaign_id)
|
||||
.filter(LabelingExportJob.action != "labeling_ml_predict")
|
||||
.order_by(LabelingExportJob.created_at.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
for ej in rows:
|
||||
_sync_export_job_from_queue(ej)
|
||||
db.flush()
|
||||
items = [ej.to_dict() for ej in rows]
|
||||
return {"items": items, "campaign_id": campaign_id}
|
||||
|
||||
|
||||
def list_labeling_assignees() -> dict[str, Any]:
|
||||
"""可指派为批次负责人的用户(标注相关角色)。"""
|
||||
role_codes = ("labeler", "internal_labeler", "vendor_labeler", "engineer", "admin")
|
||||
with session_scope() as db:
|
||||
users = (
|
||||
db.query(User)
|
||||
.filter(User.is_active.is_(True))
|
||||
.order_by(User.name)
|
||||
.all()
|
||||
)
|
||||
items = []
|
||||
for u in users:
|
||||
codes = {r.code for r in (u.roles or [])}
|
||||
if codes.intersection(role_codes):
|
||||
items.append({"id": u.id, "name": u.name or f"user-{u.id}", "roles": sorted(codes)})
|
||||
return {"items": items}
|
||||
|
||||
|
||||
def _find_batch_for_campaign_id(campaign_id: str) -> dict[str, Any] | None:
|
||||
"""由确定性 campaign_id 反查 pending / registry 批次行。"""
|
||||
wf = load_wf()
|
||||
reg = load_dms_registry()
|
||||
candidates: list[dict[str, Any]] = []
|
||||
report = get_pending_report(wf)
|
||||
candidates.extend(report.get("batches") or [])
|
||||
candidates.extend(_registry_fallback_batches(wf, reg))
|
||||
for b in candidates:
|
||||
cid = _campaign_id(
|
||||
b.get("project") or "dms",
|
||||
b.get("task") or "",
|
||||
b.get("mode"),
|
||||
b.get("batch") or "",
|
||||
b.get("location") or "inbox",
|
||||
)
|
||||
if cid == campaign_id:
|
||||
return b
|
||||
return None
|
||||
|
||||
|
||||
def ensure_campaign_record(campaign_id: str) -> None:
|
||||
"""提交/导出前保证 DB 中有 LabelingCampaign(未点「进入标注」时自动创建)。"""
|
||||
with session_scope() as db:
|
||||
if db.get(LabelingCampaign, campaign_id):
|
||||
return
|
||||
batch = _find_batch_for_campaign_id(campaign_id)
|
||||
if not batch:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
if batch.get("registry_only"):
|
||||
raise ValueError("该条目为任务模板占位,无真实 inbox 批次目录,请先送标入湖或从「进入标注」开启真实批次")
|
||||
open_campaign(
|
||||
project=batch.get("project") or "dms",
|
||||
task=batch.get("task") or "",
|
||||
batch=batch["batch"],
|
||||
mode=batch.get("mode"),
|
||||
pack=batch.get("pack"),
|
||||
location=batch.get("location") or "inbox",
|
||||
)
|
||||
|
||||
|
||||
def assign_campaign(campaign_id: str, user_id: int | None) -> dict[str, Any]:
|
||||
now = datetime.now(timezone.utc)
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
if user_id is None:
|
||||
camp.assigned_to_user_id = None
|
||||
camp.assigned_to_name = None
|
||||
else:
|
||||
user = db.get(User, user_id)
|
||||
if not user:
|
||||
raise ValueError(f"用户不存在: {user_id}")
|
||||
camp.assigned_to_user_id = user_id
|
||||
camp.assigned_to_name = user.name
|
||||
camp.updated_at = now
|
||||
db.flush()
|
||||
out = camp.to_dict()
|
||||
reg = load_dms_registry() if out.get("project") == "dms" else None
|
||||
return enrich_batch_labels(out, reg)
|
||||
|
||||
|
||||
def submit_campaign(campaign_id: str) -> dict[str, Any]:
|
||||
ensure_campaign_record(campaign_id)
|
||||
now = datetime.now(timezone.utc)
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
camp.status = "in_review"
|
||||
camp.updated_at = now
|
||||
db.flush()
|
||||
out = camp.to_dict()
|
||||
update_campaign_batch_meta_stage(camp, "in_review")
|
||||
reg = load_dms_registry() if out.get("project") == "dms" else None
|
||||
row = enrich_batch_labels(out, reg)
|
||||
row["stage"] = "in_review"
|
||||
return row
|
||||
|
||||
|
||||
def trigger_labeling_export(campaign_id: str) -> dict[str, Any]:
|
||||
row = get_campaign(campaign_id)
|
||||
if not row:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
job = enqueue_job(
|
||||
"labeling_export",
|
||||
{
|
||||
"campaign_id": campaign_id,
|
||||
"export_default": row.get("export_default"),
|
||||
"scope_key": row.get("scope_key"),
|
||||
"batch": row.get("batch"),
|
||||
},
|
||||
async_run=True,
|
||||
)
|
||||
ej = _record_export_job(campaign_id, "labeling_export", job)
|
||||
return {"ok": True, "job": job, "export_job": ej, "export_default": row.get("export_default")}
|
||||
116
platform/as_platform/labeling/vendor_import.py
Normal file
116
platform/as_platform/labeling/vendor_import.py
Normal file
@@ -0,0 +1,116 @@
|
||||
"""第三方标注回传包(ZIP)导入到 Campaign 批次目录。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.labeling.annotate import ANNOTATIONS_DIRNAME, resolve_campaign_batch_dir
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign, LabelingCampaignAccess
|
||||
|
||||
|
||||
def grant_campaign_access(
|
||||
db,
|
||||
*,
|
||||
campaign_id: str,
|
||||
principal_type: str = "role",
|
||||
principal_id: str = "vendor_labeler",
|
||||
access_role: str = "vendor",
|
||||
) -> dict:
|
||||
row = (
|
||||
db.query(LabelingCampaignAccess)
|
||||
.filter_by(campaign_id=campaign_id, principal_type=principal_type, principal_id=principal_id)
|
||||
.first()
|
||||
)
|
||||
if not row:
|
||||
row = LabelingCampaignAccess(
|
||||
campaign_id=campaign_id,
|
||||
principal_type=principal_type,
|
||||
principal_id=principal_id,
|
||||
access_role=access_role,
|
||||
)
|
||||
db.add(row)
|
||||
db.flush()
|
||||
return row.to_dict()
|
||||
|
||||
|
||||
def import_vendor_zip(campaign_id: str, raw: bytes) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
grant_campaign_access(db, campaign_id=campaign_id)
|
||||
|
||||
batch_dir.mkdir(parents=True, exist_ok=True)
|
||||
images_dir = batch_dir / "images"
|
||||
labels_dir = batch_dir / "labels"
|
||||
images_dir.mkdir(parents=True, exist_ok=True)
|
||||
labels_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
img_count = 0
|
||||
label_count = 0
|
||||
manifest: dict[str, Any] = {}
|
||||
|
||||
with zipfile.ZipFile(BytesIO(raw)) as zf:
|
||||
for info in zf.infolist():
|
||||
if info.is_dir():
|
||||
continue
|
||||
name = info.filename.replace("\\", "/").lstrip("/")
|
||||
lower = name.lower()
|
||||
if lower == "manifest.json":
|
||||
manifest = json.loads(zf.read(info))
|
||||
continue
|
||||
data = zf.read(info)
|
||||
base = Path(name).name
|
||||
if not base:
|
||||
continue
|
||||
if "/images/" in lower or lower.startswith("images/"):
|
||||
dest = images_dir / base
|
||||
dest.write_bytes(data)
|
||||
img_count += 1
|
||||
elif "/labels/" in lower or lower.startswith("labels/"):
|
||||
dest = labels_dir / base
|
||||
dest.write_bytes(data)
|
||||
label_count += 1
|
||||
elif base.endswith((".jpg", ".jpeg", ".png", ".bmp", ".webp")):
|
||||
(images_dir / base).write_bytes(data)
|
||||
img_count += 1
|
||||
elif base.endswith((".txt", ".json")):
|
||||
(labels_dir / base).write_bytes(data)
|
||||
label_count += 1
|
||||
|
||||
ann_dir = batch_dir / ANNOTATIONS_DIRNAME
|
||||
ann_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"campaign_id": campaign_id,
|
||||
"batch_path": str(batch_dir),
|
||||
"images_imported": img_count,
|
||||
"labels_imported": label_count,
|
||||
"manifest": manifest,
|
||||
}
|
||||
|
||||
|
||||
def list_registry_profiles() -> dict[str, Any]:
|
||||
from as_platform.labeling.scope import load_labeling_registry
|
||||
|
||||
reg = load_labeling_registry()
|
||||
profiles = reg.get("profiles") or {}
|
||||
items = []
|
||||
for key, prof in profiles.items():
|
||||
items.append(
|
||||
{
|
||||
"profile_key": key,
|
||||
"editor_template": prof.get("editor_template"),
|
||||
"export_default": prof.get("export_default"),
|
||||
"ml_adapter": prof.get("ml_adapter"),
|
||||
"type": prof.get("type"),
|
||||
}
|
||||
)
|
||||
return {"profiles": items, "version": reg.get("version")}
|
||||
@@ -29,10 +29,13 @@ def ping_redis() -> bool:
|
||||
|
||||
|
||||
def publish(event: str, payload: dict[str, Any]) -> None:
|
||||
r = get_redis()
|
||||
if not r:
|
||||
try:
|
||||
r = get_redis()
|
||||
if not r:
|
||||
return
|
||||
r.publish("as:events", json.dumps({"event": event, **payload}, ensure_ascii=False))
|
||||
except Exception:
|
||||
return
|
||||
r.publish("as:events", json.dumps({"event": event, **payload}, ensure_ascii=False))
|
||||
|
||||
|
||||
def push_job(job_id: str) -> None:
|
||||
|
||||
@@ -124,9 +124,10 @@ def list_training_records(
|
||||
kind: str | None = None,
|
||||
status: str | None = None,
|
||||
task: str | None = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
limit: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
jobs = list_jobs(status=status, limit=500)
|
||||
jobs = list_jobs(status=status, offset=0, limit=2000)["items"]
|
||||
records: list[dict[str, Any]] = []
|
||||
for job in jobs:
|
||||
if job.get("action") not in TRAINING_ACTIONS:
|
||||
@@ -139,9 +140,9 @@ def list_training_records(
|
||||
if task and rec.get("task") != task:
|
||||
continue
|
||||
records.append(rec)
|
||||
if len(records) >= limit:
|
||||
break
|
||||
return {"items": records, "total": len(records), "summary": _summarize(records)}
|
||||
total = len(records)
|
||||
page = records[max(0, offset) : max(0, offset) + max(1, limit)]
|
||||
return {"items": page, "total": total, "offset": offset, "limit": limit, "summary": _summarize(records)}
|
||||
|
||||
|
||||
def get_training_record(job_id: str) -> dict[str, Any] | None:
|
||||
|
||||
Reference in New Issue
Block a user