feat: HSAP platform v2 — modular navigation, quality review, audit log, world model simulation

Major changes:
- New frontend (platform/web/): Vite + React 18 + TypeScript + Tailwind
- 4-module navigation: 数据送标 / 模型管理 / 车队管理 / 系统管理
- Data catalog with charts (DMS/ADAS/Lane 3-tab view)
- Quality review workflow (标注质检): Good/Fine/Bad scoring with auto-advance
- Audit enhancements: batch operations, rejection categories, Feishu notifications
- Operation audit log (操作日志)
- World model simulation studio (仿真工坊)
- Dataset version management with snapshots and diff
- ADAS 7-class dataset integration (138K images organized + compressed)
- User management with Feishu integration and pagination
- CRUD/search/filter on all pages, card layout redesign
- PIL-optimized image overlay rendering
- Auto-snapshot on build, in_review workflow stage
- Removed embedded algorithm code (now in workspace)
This commit is contained in:
2026-06-03 11:40:21 +08:00
parent 7c43b44c57
commit e72bc061c5
5487 changed files with 979207 additions and 6197 deletions

View File

@@ -0,0 +1 @@
"""标注 Campaign 与 scope 解析。"""

View File

@@ -0,0 +1,287 @@
"""标注画布批次目录、LS 配置 XML、任务列表、标注 JSON、媒体文件。"""
from __future__ import annotations
import hashlib
import json
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from urllib.parse import quote
import yaml
from as_platform.config import WORKSPACE
from as_platform.data.batch import IMG_EXTS
from as_platform.data.core import load_wf, proj_root, resolve_pack_dir
from as_platform.db.engine import session_scope
from as_platform.db.models import LabelingCampaign, User
from as_platform.labeling.scope import (
enrich_batch_labels,
labeling_profile_key,
load_dms_registry,
load_labeling_registry,
)
ANNOTATIONS_DIRNAME = "ls_annotations"
def _label_config_dir(project: str) -> Path:
return WORKSPACE / "datasets" / project / "configs" / "label_studio"
_FALLBACK_XML = """<View>
<Image name="image" value="$image"/>
<RectangleLabels name="label" toName="image">
<Label value="object"/>
</RectangleLabels>
</View>"""
def _load_campaign(campaign_id: str) -> LabelingCampaign | None:
with session_scope() as db:
return db.get(LabelingCampaign, campaign_id)
def resolve_editor_xml(project: str, task: str, mode: str | None) -> str:
reg = load_dms_registry() if project == "dms" else None
pk = labeling_profile_key(project, task or "lane_v1", mode, reg)
prof = (load_labeling_registry().get("profiles") or {}).get(pk) or {}
default_tpl = "dam_15cls.xml" if project == "dms" else "lane_ufld_mask.xml"
template = prof.get("editor_template") or default_tpl
path = _label_config_dir(project) / template
if path.is_file():
return path.read_text(encoding="utf-8")
return _FALLBACK_XML
def resolve_campaign_batch_dir(camp: LabelingCampaign) -> Path:
wf = load_wf()
root = proj_root(wf, camp.project)
if camp.project == "dms":
reg = yaml.safe_load((root / wf["projects"]["dms"]["registry"]).read_text(encoding="utf-8"))
tcfg = reg["tasks"][camp.task]
if camp.location == "sources":
if not camp.pack:
raise ValueError("sources 批次需要 pack")
pack_dir = resolve_pack_dir("dms", root, wf, camp.pack)
src_sub = (reg.get("ingest") or {}).get("sources_subdir", "sources")
return (pack_dir / tcfg["task_dir"] / src_sub / camp.batch).resolve()
if tcfg.get("type") == "multi" and camp.mode:
from as_platform.labeling.scope import _dms_registry_api
get_mode_config, resolve_task_id, _ = _dms_registry_api()
task_r, mode_r = resolve_task_id(camp.task, camp.mode)
mcfg = get_mode_config(task_r, mode_r, reg)
inbox_rel = mcfg.get("inbox")
if inbox_rel:
return (root / inbox_rel).resolve()
mode = camp.mode
if mode:
return (root / "inbox" / camp.task / mode / camp.batch).resolve()
return (root / "inbox" / camp.task / camp.batch).resolve()
if camp.location == "pack" and camp.pack:
try:
from as_platform.data.core import resolve_pack
rel = resolve_pack("lane", root, wf, camp.pack)
return (root / rel).resolve()
except ValueError:
return (root / camp.pack).resolve()
return (root / "inbox" / camp.batch).resolve()
def _iter_batch_images(batch_dir: Path) -> list[Path]:
if not batch_dir.is_dir():
return []
candidates: list[Path] = []
search_roots = [
batch_dir / "images",
batch_dir / "images" / "train",
batch_dir,
]
seen: set[str] = set()
for root in search_roots:
if not root.is_dir():
continue
for p in sorted(root.rglob("*")):
if not p.is_file() or p.suffix not in IMG_EXTS:
continue
key = str(p.resolve())
if key in seen:
continue
seen.add(key)
candidates.append(p.resolve())
return candidates
def _task_id_for_image(image_path: Path, batch_dir: Path) -> str:
try:
rel = image_path.relative_to(batch_dir)
stem = rel.as_posix()
except ValueError:
stem = image_path.stem
return hashlib.sha256(stem.encode()).hexdigest()[:16]
def _annotations_dir(batch_dir: Path) -> Path:
d = batch_dir / "labels" / ANNOTATIONS_DIRNAME
d.mkdir(parents=True, exist_ok=True)
return d
def sync_campaign_config_xml(camp: LabelingCampaign) -> str:
"""始终与 labeling.registry + 模板文件对齐,避免 campaign 卡在旧 fallback XML。"""
xml = resolve_editor_xml(camp.project, camp.task, camp.mode)
if camp.config_xml != xml:
camp.config_xml = xml
return xml
def campaign_bootstrap(campaign_id: str) -> dict[str, Any]:
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise FileNotFoundError("campaign not found")
sync_campaign_config_xml(camp)
reg = load_dms_registry() if camp.project == "dms" else None
row = enrich_batch_labels(camp.to_dict(), reg)
row["config_xml"] = camp.config_xml
try:
batch_dir = resolve_campaign_batch_dir(camp)
row["batch_path"] = str(batch_dir)
row["image_count"] = len(_iter_batch_images(batch_dir))
except Exception as e:
row["batch_path"] = None
row["image_count"] = 0
row["batch_error"] = str(e)
return row
def campaign_tasks(
campaign_id: str,
*,
offset: int = 0,
limit: int = 50,
user: User | None = None,
assignee: str | None = None,
) -> dict[str, Any]:
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise FileNotFoundError("campaign not found")
batch_dir = resolve_campaign_batch_dir(camp)
images = _iter_batch_images(batch_dir)
from as_platform.labeling.progress import get_assigned_task_ids, user_is_coordinator
filter_ids: set[str] | None = None
if assignee == "me" and user:
filter_ids = get_assigned_task_ids(campaign_id, user.id)
if not filter_ids and not user_is_coordinator(user):
return {
"tasks": [],
"total": 0,
"offset": offset,
"limit": limit,
"hint": "暂无分配给您的任务,请联系协调员在送标工作台均分任务",
}
if filter_ids is not None:
filtered = [img for img in images if _task_id_for_image(img, batch_dir) in filter_ids]
images = filtered
total = len(images)
slice_imgs = images[offset : offset + limit]
tasks: list[dict[str, Any]] = []
for img in slice_imgs:
tid = _task_id_for_image(img, batch_dir)
try:
rel = img.relative_to(batch_dir).as_posix()
except ValueError:
rel = img.name
media_path = quote(rel, safe="/")
tasks.append(
{
"id": tid,
"data": {
"image": f"/api/v1/labeling/media/{campaign_id}/{media_path}",
},
"meta": {"filename": img.name, "relative_path": rel},
}
)
out: dict[str, Any] = {"tasks": tasks, "total": total, "offset": offset, "limit": limit}
if filter_ids is not None and user and assignee == "me":
out["my_assigned"] = len(filter_ids)
return out
def resolve_media_file(campaign_id: str, rel_path: str) -> Path:
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise FileNotFoundError("campaign not found")
batch_dir = resolve_campaign_batch_dir(camp)
clean = Path(rel_path)
if clean.is_absolute() or ".." in clean.parts:
raise PermissionError("invalid path")
target = (batch_dir / clean).resolve()
if not target.is_file() or not target.is_relative_to(batch_dir.resolve()):
raise FileNotFoundError("media not found")
return target
def get_annotation(campaign_id: str, task_id: str) -> dict[str, Any]:
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise FileNotFoundError("campaign not found")
batch_dir = resolve_campaign_batch_dir(camp)
path = _annotations_dir(batch_dir) / f"{task_id}.json"
if not path.is_file():
return {"task_id": task_id, "result": None, "annotations": []}
data = json.loads(path.read_text(encoding="utf-8"))
return data
def save_annotation(
campaign_id: str,
task_id: str,
payload: dict[str, Any],
*,
user: User | None = None,
) -> dict[str, Any]:
from as_platform.labeling.progress import assert_can_save_task, mark_task_completed
if user:
assert_can_save_task(campaign_id, task_id, user)
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise FileNotFoundError("campaign not found")
batch_dir = resolve_campaign_batch_dir(camp)
path = _annotations_dir(batch_dir) / f"{task_id}.json"
now = datetime.now(timezone.utc).isoformat()
extra: dict[str, Any] = {}
if user:
extra["completed_by_user_id"] = user.id
extra["completed_at"] = now
out = {"task_id": task_id, **payload, **extra}
path.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
if user and _annotation_has_result(path):
mark_task_completed(campaign_id, task_id, user.id)
return {"ok": True, "path": str(path)}
def _annotation_has_result(path: Path) -> bool:
if not path.is_file():
return False
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return False
result = data.get("result")
if result is None:
return False
if isinstance(result, list):
return len(result) > 0
if isinstance(result, dict):
return len(result) > 0
return bool(result)

View File

@@ -0,0 +1,81 @@
"""同步 inbox/sources 批次 batch.meta.yaml 的 stage与 Campaign 状态一致。"""
from __future__ import annotations
from pathlib import Path
from as_platform.data.batch import read_meta, write_meta
from as_platform.db.engine import session_scope
from as_platform.db.models import LabelingCampaign
from as_platform.labeling.annotate import resolve_campaign_batch_dir
def batch_has_yolo_labels(batch_dir: Path) -> bool:
"""批次是否已有导出的 YOLO txtlabels/train 或 labels 根目录)。"""
for sub in ("labels/train", "labels"):
d = batch_dir / sub
if d.is_dir() and any(d.glob("*.txt")):
return True
return False
def batch_has_lane_labels(batch_dir: Path) -> bool:
"""批次是否已有 UFLD mask 清单list/train_gt.txt + annotations/*.png"""
list_path = batch_dir / "list" / "train_gt.txt"
if not list_path.is_file():
return False
ann_dir = batch_dir / "annotations"
if not ann_dir.is_dir():
return False
return any(ann_dir.rglob("*.png"))
def update_campaign_batch_meta_stage(camp: LabelingCampaign, stage: str) -> bool:
try:
batch_dir = resolve_campaign_batch_dir(camp)
except Exception:
return False
if not batch_dir.is_dir():
return False
meta = read_meta(batch_dir) or {}
meta["stage"] = stage
meta.setdefault("project", camp.project)
meta.setdefault("task", camp.task)
meta.setdefault("batch", camp.batch)
meta.setdefault("location", camp.location or "inbox")
if camp.mode:
meta.setdefault("mode", camp.mode)
write_meta(batch_dir, meta)
return True
def update_campaign_batch_meta_stage_by_id(campaign_id: str, stage: str) -> bool:
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
return False
return update_campaign_batch_meta_stage(camp, stage)
def on_labeling_export_job_succeeded(job: dict) -> None:
"""导出 Job 成功且批次已有训练标签时进入 returned待入库"""
if job.get("action") != "labeling_export":
return
params = job.get("params") or {}
cid = params.get("campaign_id")
if not cid:
return
with session_scope() as db:
camp = db.get(LabelingCampaign, str(cid))
if not camp:
return
try:
batch_dir = resolve_campaign_batch_dir(camp)
except Exception:
return
has_labels = (
batch_has_lane_labels(batch_dir)
if camp.project == "lane"
else batch_has_yolo_labels(batch_dir)
)
if has_labels:
update_campaign_batch_meta_stage_by_id(str(cid), "returned")

View File

@@ -0,0 +1,94 @@
"""标注任务 Redis 互斥锁campaign + task 粒度)。"""
from __future__ import annotations
import json
import time
from typing import Any
from as_platform.redis.bus import get_redis
LOCK_TTL_SEC = 300
_LOCK_PREFIX = "labeling:lock:"
# API 进程内回退(无 Redis 时)
_memory: dict[str, dict[str, Any]] = {}
def _key(campaign_id: str, task_id: str) -> str:
return f"{_LOCK_PREFIX}{campaign_id}:{task_id}"
def _parse_holder(raw: str | None) -> dict[str, Any] | None:
if not raw:
return None
try:
return json.loads(raw)
except json.JSONDecodeError:
return {"name": raw}
def acquire_lock(campaign_id: str, task_id: str, *, user_id: int, user_name: str) -> dict[str, Any]:
payload = json.dumps({"user_id": user_id, "name": user_name}, ensure_ascii=False)
r = get_redis()
if not r:
now = time.time()
mem = _memory.get(_key(campaign_id, task_id))
if mem and mem.get("user_id") != user_id and now < mem.get("expires_at", 0):
return {"ok": False, "holder": mem.get("name"), "user_id": mem.get("user_id")}
_memory[_key(campaign_id, task_id)] = {
"user_id": user_id,
"name": user_name,
"expires_at": now + LOCK_TTL_SEC,
}
return {"ok": True, "holder": user_name, "ttl_sec": LOCK_TTL_SEC, "backend": "memory"}
key = _key(campaign_id, task_id)
if r.set(key, payload, nx=True, ex=LOCK_TTL_SEC):
return {"ok": True, "holder": user_name, "ttl_sec": LOCK_TTL_SEC, "backend": "redis"}
existing = _parse_holder(r.get(key))
if existing and existing.get("user_id") == user_id:
r.expire(key, LOCK_TTL_SEC)
return {"ok": True, "holder": user_name, "ttl_sec": LOCK_TTL_SEC, "renewed": True, "backend": "redis"}
return {
"ok": False,
"holder": (existing or {}).get("name"),
"user_id": (existing or {}).get("user_id"),
"backend": "redis",
}
def release_lock(campaign_id: str, task_id: str, *, user_id: int) -> dict[str, Any]:
r = get_redis()
if not r:
key = _key(campaign_id, task_id)
mem = _memory.get(key)
if mem and mem.get("user_id") == user_id:
_memory.pop(key, None)
return {"ok": True, "released": True, "backend": "memory"}
return {"ok": True, "released": False, "backend": "memory"}
key = _key(campaign_id, task_id)
existing = _parse_holder(r.get(key))
if not existing:
return {"ok": True, "released": False, "backend": "redis"}
if existing.get("user_id") != user_id:
return {"ok": False, "holder": existing.get("name"), "backend": "redis"}
r.delete(key)
return {"ok": True, "released": True, "backend": "redis"}
def renew_lock(campaign_id: str, task_id: str, *, user_id: int) -> dict[str, Any]:
r = get_redis()
if not r:
key = _key(campaign_id, task_id)
mem = _memory.get(key)
if mem and mem.get("user_id") == user_id:
mem["expires_at"] = time.time() + LOCK_TTL_SEC
return {"ok": True, "ttl_sec": LOCK_TTL_SEC, "backend": "memory"}
return {"ok": False, "backend": "memory"}
key = _key(campaign_id, task_id)
existing = _parse_holder(r.get(key))
if not existing or existing.get("user_id") != user_id:
return {"ok": False, "holder": (existing or {}).get("name"), "backend": "redis"}
r.expire(key, LOCK_TTL_SEC)
return {"ok": True, "ttl_sec": LOCK_TTL_SEC, "backend": "redis"}

View File

@@ -0,0 +1,313 @@
"""标注进度统计与按人分包。"""
from __future__ import annotations
import json
from collections import defaultdict
from datetime import datetime, timezone
from typing import Any
from as_platform.db.engine import session_scope
from as_platform.db.models import LabelingCampaign, LabelingTaskAssignment, User
from as_platform.labeling.annotate import (
_annotations_dir,
_iter_batch_images,
_task_id_for_image,
resolve_campaign_batch_dir,
)
COORDINATOR_ROLES = frozenset({"labeler", "admin", "engineer", "reviewer"})
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
def user_role_codes(user: User) -> set[str]:
return {r.code for r in (user.roles or [])}
def user_is_coordinator(user: User) -> bool:
codes = user_role_codes(user)
if codes & COORDINATOR_ROLES:
return True
perms: set[str] = set()
for r in user.roles or []:
for p in r.permissions or []:
if p.code:
perms.add(p.code)
return "*" in perms or "write:labeling_assign" in perms
def list_campaign_task_ids(campaign_id: str) -> list[str]:
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise FileNotFoundError("campaign not found")
batch_dir = resolve_campaign_batch_dir(camp)
images = _iter_batch_images(batch_dir)
return [_task_id_for_image(img, batch_dir) for img in images]
def _annotation_has_result(path) -> bool:
if not path.is_file():
return False
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return False
result = data.get("result")
if result is None:
return False
if isinstance(result, list):
return len(result) > 0
if isinstance(result, dict):
return len(result) > 0
return bool(result)
def count_completed_tasks(batch_dir) -> set[str]:
ann_dir = _annotations_dir(batch_dir)
done: set[str] = set()
if not ann_dir.is_dir():
return done
for p in ann_dir.glob("*.json"):
if _annotation_has_result(p):
done.add(p.stem)
return done
def _user_name_map(db, user_ids: set[int]) -> dict[int, str]:
if not user_ids:
return {}
rows = db.query(User).filter(User.id.in_(user_ids)).all()
return {u.id: u.name or f"user-{u.id}" for u in rows}
def campaign_progress(campaign_id: str) -> dict[str, Any]:
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise FileNotFoundError("campaign not found")
batch_dir = resolve_campaign_batch_dir(camp)
all_ids = [_task_id_for_image(img, batch_dir) for img in _iter_batch_images(batch_dir)]
completed_ids = count_completed_tasks(batch_dir)
rows = (
db.query(LabelingTaskAssignment)
.filter(LabelingTaskAssignment.campaign_id == campaign_id)
.all()
)
user_ids = {r.user_id for r in rows} | {r.completed_by_user_id for r in rows if r.completed_by_user_id}
names = _user_name_map(db, user_ids)
total = len(all_ids)
completed = len(completed_ids)
assigned = len(rows)
by_user_agg: dict[int, dict[str, int]] = defaultdict(lambda: {"assigned": 0, "completed": 0})
for r in rows:
by_user_agg[r.user_id]["assigned"] += 1
if r.completed_at or r.task_id in completed_ids:
by_user_agg[r.user_id]["completed"] += 1
by_user = []
for uid, stats in sorted(by_user_agg.items(), key=lambda x: names.get(x[0], "")):
a = stats["assigned"]
c = stats["completed"]
by_user.append(
{
"user_id": uid,
"name": names.get(uid, f"user-{uid}"),
"assigned": a,
"completed": c,
"percent": round(100.0 * c / a, 1) if a else 0.0,
}
)
return {
"campaign_id": campaign_id,
"total_tasks": total,
"completed_tasks": completed,
"assigned_tasks": assigned,
"unassigned_tasks": max(0, total - assigned),
"percent": round(100.0 * completed / total, 1) if total else 0.0,
"by_user": by_user,
}
def campaign_progress_summary(campaign_id: str) -> dict[str, int]:
try:
p = campaign_progress(campaign_id)
return {
"total_tasks": p["total_tasks"],
"completed_tasks": p["completed_tasks"],
"assigned_tasks": p["assigned_tasks"],
}
except FileNotFoundError:
return {"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0}
def get_assigned_task_ids(campaign_id: str, user_id: int | None = None) -> set[str]:
with session_scope() as db:
q = db.query(LabelingTaskAssignment.task_id).filter(
LabelingTaskAssignment.campaign_id == campaign_id
)
if user_id is not None:
q = q.filter(LabelingTaskAssignment.user_id == user_id)
return {row[0] for row in q.all()}
def _assign_result(campaign_id: str, created: int) -> dict[str, Any]:
prog = campaign_progress(campaign_id)
return {"assigned": created, "by_user": prog["by_user"], "progress": prog}
def assign_tasks_even(
campaign_id: str,
user_ids: list[int],
*,
assigned_by_user_id: int,
) -> dict[str, Any]:
if not user_ids:
raise ValueError("user_ids 不能为空")
now = _utcnow()
all_ids = list_campaign_task_ids(campaign_id)
with session_scope() as db:
existing = {
r.task_id
for r in db.query(LabelingTaskAssignment)
.filter(LabelingTaskAssignment.campaign_id == campaign_id)
.all()
}
unassigned = [tid for tid in all_ids if tid not in existing]
users = db.query(User).filter(User.id.in_(user_ids)).all()
if len(users) != len(set(user_ids)):
raise ValueError("存在无效 user_id")
created = 0
for i, tid in enumerate(unassigned):
uid = user_ids[i % len(user_ids)]
db.add(
LabelingTaskAssignment(
campaign_id=campaign_id,
task_id=tid,
user_id=uid,
assigned_by_user_id=assigned_by_user_id,
assigned_at=now,
)
)
created += 1
db.flush()
return _assign_result(campaign_id, created)
def assign_tasks_explicit(
campaign_id: str,
items: list[dict[str, Any]],
*,
assigned_by_user_id: int,
) -> dict[str, Any]:
now = _utcnow()
all_set = set(list_campaign_task_ids(campaign_id))
created = 0
with session_scope() as db:
existing = {
r.task_id
for r in db.query(LabelingTaskAssignment)
.filter(LabelingTaskAssignment.campaign_id == campaign_id)
.all()
}
for item in items:
uid = int(item["user_id"])
user = db.get(User, uid)
if not user:
raise ValueError(f"用户不存在: {uid}")
for tid in item.get("task_ids") or []:
if tid not in all_set:
raise ValueError(f"无效 task_id: {tid}")
if tid in existing:
continue
db.add(
LabelingTaskAssignment(
campaign_id=campaign_id,
task_id=tid,
user_id=uid,
assigned_by_user_id=assigned_by_user_id,
assigned_at=now,
)
)
existing.add(tid)
created += 1
db.flush()
return _assign_result(campaign_id, created)
def reassign_task(campaign_id: str, task_id: str, user_id: int) -> dict[str, Any]:
with session_scope() as db:
row = (
db.query(LabelingTaskAssignment)
.filter(
LabelingTaskAssignment.campaign_id == campaign_id,
LabelingTaskAssignment.task_id == task_id,
)
.first()
)
if not row:
raise FileNotFoundError("assignment not found")
user = db.get(User, user_id)
if not user:
raise ValueError(f"用户不存在: {user_id}")
row.user_id = user_id
db.flush()
return row.to_dict()
def release_task_assignment(campaign_id: str, task_id: str) -> dict[str, Any]:
with session_scope() as db:
row = (
db.query(LabelingTaskAssignment)
.filter(
LabelingTaskAssignment.campaign_id == campaign_id,
LabelingTaskAssignment.task_id == task_id,
)
.first()
)
if not row:
raise FileNotFoundError("assignment not found")
db.delete(row)
db.flush()
return {"ok": True, "released": task_id}
def assert_can_save_task(campaign_id: str, task_id: str, user: User) -> None:
if user_is_coordinator(user):
return
codes = user_role_codes(user)
if "vendor_labeler" in codes:
return
with session_scope() as db:
row = (
db.query(LabelingTaskAssignment)
.filter(
LabelingTaskAssignment.campaign_id == campaign_id,
LabelingTaskAssignment.task_id == task_id,
)
.first()
)
if not row:
raise PermissionError("该图未分配给您,请联系协调员分包")
if row.user_id != user.id:
raise PermissionError("该图已分配给其他标注员")
def mark_task_completed(campaign_id: str, task_id: str, user_id: int) -> None:
now = _utcnow()
with session_scope() as db:
row = (
db.query(LabelingTaskAssignment)
.filter(
LabelingTaskAssignment.campaign_id == campaign_id,
LabelingTaskAssignment.task_id == task_id,
)
.first()
)
if row and not row.completed_at:
row.completed_at = now
row.completed_by_user_id = user_id

View File

@@ -0,0 +1,86 @@
"""CatalogScope 与 registry 对齐。"""
from __future__ import annotations
from typing import Any
import yaml
from as_platform.config import WORKSPACE
DOMAIN_LABELS = {"dms": "舱内 DMS", "forward": "前向 ADAS"}
def format_scope_key(project: str, task: str, mode: str | None = None) -> str:
if project == "lane":
return f"lane:{task}"
if mode:
return f"dms:{task}:{mode}"
return f"dms:{task}"
def _dms_registry_api():
import sys
from pathlib import Path
p = WORKSPACE / "datasets" / "dms" / "scripts"
if str(p) not in sys.path:
sys.path.insert(0, str(p))
from task_registry import get_mode_config, resolve_task_id, train_yaml_key
return get_mode_config, resolve_task_id, train_yaml_key
def load_dms_registry() -> dict:
path = WORKSPACE / "datasets" / "dms" / "datasets.registry.yaml"
return yaml.safe_load(path.read_text(encoding="utf-8"))
def load_labeling_registry() -> dict[str, Any]:
path = WORKSPACE / "datasets" / "labeling.registry.yaml"
if not path.is_file():
return {"profiles": {}}
return yaml.safe_load(path.read_text(encoding="utf-8")) or {"profiles": {}}
def labeling_profile_key(project: str, task: str, mode: str | None, reg: dict | None = None) -> str:
if project == "lane":
return f"lane__{task}"
get_mode_config, resolve_task_id, train_yaml_key = _dms_registry_api()
reg = reg or load_dms_registry()
task, mode = resolve_task_id(task, mode)
return train_yaml_key(task, mode, reg)
def enrich_batch_labels(batch: dict[str, Any], reg: dict | None = None) -> dict[str, Any]:
project = batch.get("project") or "dms"
task = batch.get("task") or ""
mode = batch.get("mode")
out = dict(batch)
out["scope_key"] = format_scope_key(project, task, mode)
if project == "dms":
reg = reg or load_dms_registry()
get_mode_config, resolve_task_id, _ = _dms_registry_api()
try:
task_r, mode_r = resolve_task_id(task, mode)
mcfg = get_mode_config(task_r, mode_r, reg)
domain = mcfg.get("domain") or "dms"
out["domain"] = domain
out["domain_label"] = DOMAIN_LABELS.get(domain, domain)
out["task_label"] = mcfg.get("label") or task
if mode_r:
modes = (reg.get("tasks") or {}).get(task_r, {}).get("modes") or {}
out["mode_label"] = (modes.get(mode_r) or {}).get("label") or mode_r
except Exception:
out["domain"] = "dms"
out["domain_label"] = DOMAIN_LABELS["dms"]
else:
out["domain_label"] = "车道线 Lane"
out["task_label"] = task
try:
pk = labeling_profile_key(project, task, mode, reg if project == "dms" else None)
out["labeling_profile"] = pk
prof = (load_labeling_registry().get("profiles") or {}).get(pk)
if prof:
out["export_default"] = prof.get("export_default")
out["ml_adapter"] = prof.get("ml_adapter")
except Exception:
out["labeling_profile"] = None
return out

View File

@@ -0,0 +1,401 @@
"""Campaign 与 pending 批次合并列表。"""
from __future__ import annotations
import hashlib
import json
import uuid
from datetime import datetime, timezone
from typing import Any
from as_platform.config import WORKSPACE
from as_platform.data.core import get_pending_report, load_wf
from as_platform.db.engine import session_scope
from as_platform.db.models import LabelingCampaign, LabelingExportJob, User
from as_platform.jobs.queue import enqueue_job, get_job
from as_platform.labeling.annotate import resolve_editor_xml, sync_campaign_config_xml
from as_platform.labeling.batch_stage import (
on_labeling_export_job_succeeded,
update_campaign_batch_meta_stage,
)
from as_platform.labeling.scope import (
enrich_batch_labels,
format_scope_key,
load_dms_registry,
load_labeling_registry,
)
def _campaign_id(project: str, task: str, mode: str | None, batch: str, location: str) -> str:
sk = format_scope_key(project, task, mode)
raw = f"{sk}:{batch}:{location}"
return hashlib.sha256(raw.encode()).hexdigest()[:20]
def _parse_scope_key(scope_key: str) -> tuple[str, str, str | None]:
parts = scope_key.split(":")
if parts[0] == "lane":
return "lane", parts[1] if len(parts) > 1 else "lane_v1", None
if len(parts) >= 3:
return "dms", parts[1], parts[2]
if len(parts) == 2:
return "dms", parts[1], None
return "dms", parts[-1], None
def _registry_fallback_batches(wf: dict, reg: dict) -> list[dict[str, Any]]:
"""labeling.registry 中有配置但 pending 未扫到的批次(如空 inbox"""
from pathlib import Path
from as_platform.data.batch import enrich_batch
from as_platform.data.core import proj_root
profiles = load_labeling_registry().get("profiles") or {}
rows: list[dict[str, Any]] = []
dms_root = proj_root(wf, "dms")
for _pk, prof in profiles.items():
scope_key = prof.get("scope_key") or ""
project, task, mode = _parse_scope_key(scope_key)
if project != "dms":
continue
batch = mode or task
batch_dir = None
if mode:
try:
import sys
scripts = WORKSPACE / "datasets" / "dms" / "scripts"
if str(scripts) not in sys.path:
sys.path.insert(0, str(scripts))
from task_registry import inbox_dir, resolve_task_id
task_r, mode_r = resolve_task_id(task, mode)
batch_dir = inbox_dir(dms_root, task_r, mode_r, reg)
except Exception:
batch_dir = dms_root / "inbox" / task / mode
else:
batch_dir = dms_root / "inbox" / task / batch
if not isinstance(batch_dir, Path) or not batch_dir.is_dir():
row = {
"project": project,
"task": task,
"mode": mode,
"batch": batch,
"stage": "raw_pool",
"location": "inbox",
"path": str(batch_dir) if batch_dir else "",
"counts": {"images": 0, "labels": 0},
"registry_only": True,
}
else:
row = enrich_batch(
batch_dir,
project=project,
task=task,
pack=None,
batch=batch,
location="inbox",
)
row["mode"] = mode
row["scope_key"] = scope_key
rows.append(row)
return rows
def list_labeling_batches(
*,
stage: str | None = None,
offset: int = 0,
limit: int = 20,
) -> dict[str, Any]:
wf = load_wf()
report = get_pending_report(wf)
reg = load_dms_registry()
items: list[dict[str, Any]] = []
seen: set[str] = set()
allowed_stages = ("raw_pool", "out_for_labeling", "returned", "labeling_submitted", "in_review", "review_approved", "review_rejected")
def _append(b: dict[str, Any]) -> None:
if stage and b.get("stage") != stage:
return
if b.get("stage") not in allowed_stages:
return
row = enrich_batch_labels(b, reg)
cid = _campaign_id(
row["project"], row.get("task") or "", row.get("mode"), row["batch"], row.get("location") or "inbox"
)
key = f"{cid}"
if key in seen:
return
seen.add(key)
with session_scope() as db:
camp = db.get(LabelingCampaign, cid)
status = camp.status if camp else "not_opened"
if camp:
row["assigned_to_user_id"] = camp.assigned_to_user_id
row["assigned_to_name"] = camp.assigned_to_name
row["campaign_id"] = cid
row["campaign_status"] = status
if camp and status in ("in_progress", "labeling_submitted"):
try:
from as_platform.labeling.progress import campaign_progress_summary
row.update(campaign_progress_summary(cid))
except Exception:
row.update({"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0})
items.append(row)
for b in report.get("batches", []):
_append(b)
for b in _registry_fallback_batches(wf, reg):
_append(b)
total = len(items)
page = items[max(0, offset) : max(0, offset) + max(1, limit)]
return {
"items": page,
"total": total,
"offset": offset,
"limit": limit,
"updated_at": report.get("updated_at"),
}
def open_campaign(
*,
project: str,
task: str,
batch: str,
mode: str | None = None,
pack: str | None = None,
location: str = "inbox",
) -> dict[str, Any]:
cid = _campaign_id(project, task, mode, batch, location)
config_xml = resolve_editor_xml(project, task, mode)
now = datetime.now(timezone.utc)
with session_scope() as db:
camp = db.get(LabelingCampaign, cid)
if not camp:
camp = LabelingCampaign(
id=cid,
project=project,
task=task,
mode=mode,
batch=batch,
pack=pack,
location=location,
status="in_progress",
config_xml=config_xml,
created_at=now,
updated_at=now,
)
db.add(camp)
else:
camp.status = "in_progress"
camp.updated_at = now
sync_campaign_config_xml(camp)
db.flush()
out = camp.to_dict()
out["config_xml"] = camp.config_xml
update_campaign_batch_meta_stage(camp, "out_for_labeling")
reg = load_dms_registry() if project == "dms" else None
row = enrich_batch_labels(out, reg)
row["stage"] = "out_for_labeling"
return row
def get_campaign(campaign_id: str) -> dict[str, Any] | None:
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
return None
row = camp.to_dict()
row["config_xml"] = camp.config_xml
reg = load_dms_registry() if row.get("project") == "dms" else None
return enrich_batch_labels(row, reg)
def _export_job_id() -> str:
return f"lej-{uuid.uuid4().hex[:16]}"
def _record_export_job(campaign_id: str, action: str, job: dict[str, Any]) -> dict[str, Any]:
ej_id = _export_job_id()
job_id = job.get("id")
now = datetime.now(timezone.utc)
with session_scope() as db:
ej = LabelingExportJob(
id=ej_id,
campaign_id=campaign_id,
action=action,
job_id=job_id,
status=job.get("status") or "queued",
created_at=now,
)
db.add(ej)
out = get_export_job(ej_id)
return out or {"id": ej_id, "campaign_id": campaign_id, "action": action, "job_id": job_id}
def _sync_export_job_from_queue(ej: LabelingExportJob) -> None:
if not ej.job_id:
return
job = get_job(ej.job_id)
if not job:
return
ej.status = job.get("status") or ej.status
if job.get("finished_at"):
try:
ej.finished_at = datetime.fromisoformat(str(job["finished_at"]).replace("Z", "+00:00"))
except Exception:
pass
if job.get("result") is not None:
ej.result_json = json.dumps(job.get("result"), ensure_ascii=False)
if ej.action == "labeling_export" and ej.status in ("succeeded", "completed"):
on_labeling_export_job_succeeded(
{"action": "labeling_export", "params": {"campaign_id": ej.campaign_id}}
)
def get_export_job(export_job_id: str) -> dict[str, Any] | None:
with session_scope() as db:
ej = db.get(LabelingExportJob, export_job_id)
if not ej:
return None
_sync_export_job_from_queue(ej)
db.flush()
return ej.to_dict()
def list_campaign_export_jobs(campaign_id: str, *, limit: int = 30) -> dict[str, Any]:
with session_scope() as db:
rows = (
db.query(LabelingExportJob)
.filter_by(campaign_id=campaign_id)
.filter(LabelingExportJob.action != "labeling_ml_predict")
.order_by(LabelingExportJob.created_at.desc())
.limit(limit)
.all()
)
for ej in rows:
_sync_export_job_from_queue(ej)
db.flush()
items = [ej.to_dict() for ej in rows]
return {"items": items, "campaign_id": campaign_id}
def list_labeling_assignees() -> dict[str, Any]:
"""可指派为批次负责人的用户(标注相关角色)。"""
role_codes = ("labeler", "internal_labeler", "vendor_labeler", "engineer", "admin")
with session_scope() as db:
users = (
db.query(User)
.filter(User.is_active.is_(True))
.order_by(User.name)
.all()
)
items = []
for u in users:
codes = {r.code for r in (u.roles or [])}
if codes.intersection(role_codes):
items.append({"id": u.id, "name": u.name or f"user-{u.id}", "roles": sorted(codes)})
return {"items": items}
def _find_batch_for_campaign_id(campaign_id: str) -> dict[str, Any] | None:
"""由确定性 campaign_id 反查 pending / registry 批次行。"""
wf = load_wf()
reg = load_dms_registry()
candidates: list[dict[str, Any]] = []
report = get_pending_report(wf)
candidates.extend(report.get("batches") or [])
candidates.extend(_registry_fallback_batches(wf, reg))
for b in candidates:
cid = _campaign_id(
b.get("project") or "dms",
b.get("task") or "",
b.get("mode"),
b.get("batch") or "",
b.get("location") or "inbox",
)
if cid == campaign_id:
return b
return None
def ensure_campaign_record(campaign_id: str) -> None:
"""提交/导出前保证 DB 中有 LabelingCampaign未点「进入标注」时自动创建"""
with session_scope() as db:
if db.get(LabelingCampaign, campaign_id):
return
batch = _find_batch_for_campaign_id(campaign_id)
if not batch:
raise FileNotFoundError("campaign not found")
if batch.get("registry_only"):
raise ValueError("该条目为任务模板占位,无真实 inbox 批次目录,请先送标入湖或从「进入标注」开启真实批次")
open_campaign(
project=batch.get("project") or "dms",
task=batch.get("task") or "",
batch=batch["batch"],
mode=batch.get("mode"),
pack=batch.get("pack"),
location=batch.get("location") or "inbox",
)
def assign_campaign(campaign_id: str, user_id: int | None) -> dict[str, Any]:
now = datetime.now(timezone.utc)
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise FileNotFoundError("campaign not found")
if user_id is None:
camp.assigned_to_user_id = None
camp.assigned_to_name = None
else:
user = db.get(User, user_id)
if not user:
raise ValueError(f"用户不存在: {user_id}")
camp.assigned_to_user_id = user_id
camp.assigned_to_name = user.name
camp.updated_at = now
db.flush()
out = camp.to_dict()
reg = load_dms_registry() if out.get("project") == "dms" else None
return enrich_batch_labels(out, reg)
def submit_campaign(campaign_id: str) -> dict[str, Any]:
ensure_campaign_record(campaign_id)
now = datetime.now(timezone.utc)
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise FileNotFoundError("campaign not found")
camp.status = "in_review"
camp.updated_at = now
db.flush()
out = camp.to_dict()
update_campaign_batch_meta_stage(camp, "in_review")
reg = load_dms_registry() if out.get("project") == "dms" else None
row = enrich_batch_labels(out, reg)
row["stage"] = "in_review"
return row
def trigger_labeling_export(campaign_id: str) -> dict[str, Any]:
row = get_campaign(campaign_id)
if not row:
raise FileNotFoundError("campaign not found")
job = enqueue_job(
"labeling_export",
{
"campaign_id": campaign_id,
"export_default": row.get("export_default"),
"scope_key": row.get("scope_key"),
"batch": row.get("batch"),
},
async_run=True,
)
ej = _record_export_job(campaign_id, "labeling_export", job)
return {"ok": True, "job": job, "export_job": ej, "export_default": row.get("export_default")}

View File

@@ -0,0 +1,116 @@
"""第三方标注回传包ZIP导入到 Campaign 批次目录。"""
from __future__ import annotations
import json
import shutil
import zipfile
from io import BytesIO
from pathlib import Path
from typing import Any
from as_platform.labeling.annotate import ANNOTATIONS_DIRNAME, resolve_campaign_batch_dir
from as_platform.db.engine import session_scope
from as_platform.db.models import LabelingCampaign, LabelingCampaignAccess
def grant_campaign_access(
db,
*,
campaign_id: str,
principal_type: str = "role",
principal_id: str = "vendor_labeler",
access_role: str = "vendor",
) -> dict:
row = (
db.query(LabelingCampaignAccess)
.filter_by(campaign_id=campaign_id, principal_type=principal_type, principal_id=principal_id)
.first()
)
if not row:
row = LabelingCampaignAccess(
campaign_id=campaign_id,
principal_type=principal_type,
principal_id=principal_id,
access_role=access_role,
)
db.add(row)
db.flush()
return row.to_dict()
def import_vendor_zip(campaign_id: str, raw: bytes) -> dict[str, Any]:
with session_scope() as db:
camp = db.get(LabelingCampaign, campaign_id)
if not camp:
raise FileNotFoundError("campaign not found")
batch_dir = resolve_campaign_batch_dir(camp)
grant_campaign_access(db, campaign_id=campaign_id)
batch_dir.mkdir(parents=True, exist_ok=True)
images_dir = batch_dir / "images"
labels_dir = batch_dir / "labels"
images_dir.mkdir(parents=True, exist_ok=True)
labels_dir.mkdir(parents=True, exist_ok=True)
img_count = 0
label_count = 0
manifest: dict[str, Any] = {}
with zipfile.ZipFile(BytesIO(raw)) as zf:
for info in zf.infolist():
if info.is_dir():
continue
name = info.filename.replace("\\", "/").lstrip("/")
lower = name.lower()
if lower == "manifest.json":
manifest = json.loads(zf.read(info))
continue
data = zf.read(info)
base = Path(name).name
if not base:
continue
if "/images/" in lower or lower.startswith("images/"):
dest = images_dir / base
dest.write_bytes(data)
img_count += 1
elif "/labels/" in lower or lower.startswith("labels/"):
dest = labels_dir / base
dest.write_bytes(data)
label_count += 1
elif base.endswith((".jpg", ".jpeg", ".png", ".bmp", ".webp")):
(images_dir / base).write_bytes(data)
img_count += 1
elif base.endswith((".txt", ".json")):
(labels_dir / base).write_bytes(data)
label_count += 1
ann_dir = batch_dir / ANNOTATIONS_DIRNAME
ann_dir.mkdir(parents=True, exist_ok=True)
return {
"ok": True,
"campaign_id": campaign_id,
"batch_path": str(batch_dir),
"images_imported": img_count,
"labels_imported": label_count,
"manifest": manifest,
}
def list_registry_profiles() -> dict[str, Any]:
from as_platform.labeling.scope import load_labeling_registry
reg = load_labeling_registry()
profiles = reg.get("profiles") or {}
items = []
for key, prof in profiles.items():
items.append(
{
"profile_key": key,
"editor_template": prof.get("editor_template"),
"export_default": prof.get("export_default"),
"ml_adapter": prof.get("ml_adapter"),
"type": prof.get("type"),
}
)
return {"profiles": items, "version": reg.get("version")}