feat: HSAP platform v2 — modular navigation, quality review, audit log, world model simulation
Major changes: - New frontend (platform/web/): Vite + React 18 + TypeScript + Tailwind - 4-module navigation: 数据送标 / 模型管理 / 车队管理 / 系统管理 - Data catalog with charts (DMS/ADAS/Lane 3-tab view) - Quality review workflow (标注质检): Good/Fine/Bad scoring with auto-advance - Audit enhancements: batch operations, rejection categories, Feishu notifications - Operation audit log (操作日志) - World model simulation studio (仿真工坊) - Dataset version management with snapshots and diff - ADAS 7-class dataset integration (138K images organized + compressed) - User management with Feishu integration and pagination - CRUD/search/filter on all pages, card layout redesign - PIL-optimized image overlay rendering - Auto-snapshot on build, in_review workflow stage - Removed embedded algorithm code (now in workspace)
This commit is contained in:
1
platform/as_platform/labeling/__init__.py
Normal file
1
platform/as_platform/labeling/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""标注 Campaign 与 scope 解析。"""
|
||||
287
platform/as_platform/labeling/annotate.py
Normal file
287
platform/as_platform/labeling/annotate.py
Normal file
@@ -0,0 +1,287 @@
|
||||
"""标注画布:批次目录、LS 配置 XML、任务列表、标注 JSON、媒体文件。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import yaml
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
from as_platform.data.batch import IMG_EXTS
|
||||
from as_platform.data.core import load_wf, proj_root, resolve_pack_dir
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign, User
|
||||
from as_platform.labeling.scope import (
|
||||
enrich_batch_labels,
|
||||
labeling_profile_key,
|
||||
load_dms_registry,
|
||||
load_labeling_registry,
|
||||
)
|
||||
|
||||
ANNOTATIONS_DIRNAME = "ls_annotations"
|
||||
|
||||
|
||||
def _label_config_dir(project: str) -> Path:
|
||||
return WORKSPACE / "datasets" / project / "configs" / "label_studio"
|
||||
|
||||
_FALLBACK_XML = """<View>
|
||||
<Image name="image" value="$image"/>
|
||||
<RectangleLabels name="label" toName="image">
|
||||
<Label value="object"/>
|
||||
</RectangleLabels>
|
||||
</View>"""
|
||||
|
||||
|
||||
def _load_campaign(campaign_id: str) -> LabelingCampaign | None:
|
||||
with session_scope() as db:
|
||||
return db.get(LabelingCampaign, campaign_id)
|
||||
|
||||
|
||||
def resolve_editor_xml(project: str, task: str, mode: str | None) -> str:
|
||||
reg = load_dms_registry() if project == "dms" else None
|
||||
pk = labeling_profile_key(project, task or "lane_v1", mode, reg)
|
||||
prof = (load_labeling_registry().get("profiles") or {}).get(pk) or {}
|
||||
default_tpl = "dam_15cls.xml" if project == "dms" else "lane_ufld_mask.xml"
|
||||
template = prof.get("editor_template") or default_tpl
|
||||
path = _label_config_dir(project) / template
|
||||
if path.is_file():
|
||||
return path.read_text(encoding="utf-8")
|
||||
return _FALLBACK_XML
|
||||
|
||||
|
||||
def resolve_campaign_batch_dir(camp: LabelingCampaign) -> Path:
|
||||
wf = load_wf()
|
||||
root = proj_root(wf, camp.project)
|
||||
if camp.project == "dms":
|
||||
reg = yaml.safe_load((root / wf["projects"]["dms"]["registry"]).read_text(encoding="utf-8"))
|
||||
tcfg = reg["tasks"][camp.task]
|
||||
if camp.location == "sources":
|
||||
if not camp.pack:
|
||||
raise ValueError("sources 批次需要 pack")
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, camp.pack)
|
||||
src_sub = (reg.get("ingest") or {}).get("sources_subdir", "sources")
|
||||
return (pack_dir / tcfg["task_dir"] / src_sub / camp.batch).resolve()
|
||||
if tcfg.get("type") == "multi" and camp.mode:
|
||||
from as_platform.labeling.scope import _dms_registry_api
|
||||
|
||||
get_mode_config, resolve_task_id, _ = _dms_registry_api()
|
||||
task_r, mode_r = resolve_task_id(camp.task, camp.mode)
|
||||
mcfg = get_mode_config(task_r, mode_r, reg)
|
||||
inbox_rel = mcfg.get("inbox")
|
||||
if inbox_rel:
|
||||
return (root / inbox_rel).resolve()
|
||||
mode = camp.mode
|
||||
if mode:
|
||||
return (root / "inbox" / camp.task / mode / camp.batch).resolve()
|
||||
return (root / "inbox" / camp.task / camp.batch).resolve()
|
||||
if camp.location == "pack" and camp.pack:
|
||||
try:
|
||||
from as_platform.data.core import resolve_pack
|
||||
|
||||
rel = resolve_pack("lane", root, wf, camp.pack)
|
||||
return (root / rel).resolve()
|
||||
except ValueError:
|
||||
return (root / camp.pack).resolve()
|
||||
return (root / "inbox" / camp.batch).resolve()
|
||||
|
||||
|
||||
def _iter_batch_images(batch_dir: Path) -> list[Path]:
|
||||
if not batch_dir.is_dir():
|
||||
return []
|
||||
candidates: list[Path] = []
|
||||
search_roots = [
|
||||
batch_dir / "images",
|
||||
batch_dir / "images" / "train",
|
||||
batch_dir,
|
||||
]
|
||||
seen: set[str] = set()
|
||||
for root in search_roots:
|
||||
if not root.is_dir():
|
||||
continue
|
||||
for p in sorted(root.rglob("*")):
|
||||
if not p.is_file() or p.suffix not in IMG_EXTS:
|
||||
continue
|
||||
key = str(p.resolve())
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
candidates.append(p.resolve())
|
||||
return candidates
|
||||
|
||||
|
||||
def _task_id_for_image(image_path: Path, batch_dir: Path) -> str:
|
||||
try:
|
||||
rel = image_path.relative_to(batch_dir)
|
||||
stem = rel.as_posix()
|
||||
except ValueError:
|
||||
stem = image_path.stem
|
||||
return hashlib.sha256(stem.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def _annotations_dir(batch_dir: Path) -> Path:
|
||||
d = batch_dir / "labels" / ANNOTATIONS_DIRNAME
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def sync_campaign_config_xml(camp: LabelingCampaign) -> str:
|
||||
"""始终与 labeling.registry + 模板文件对齐,避免 campaign 卡在旧 fallback XML。"""
|
||||
xml = resolve_editor_xml(camp.project, camp.task, camp.mode)
|
||||
if camp.config_xml != xml:
|
||||
camp.config_xml = xml
|
||||
return xml
|
||||
|
||||
|
||||
def campaign_bootstrap(campaign_id: str) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
sync_campaign_config_xml(camp)
|
||||
reg = load_dms_registry() if camp.project == "dms" else None
|
||||
row = enrich_batch_labels(camp.to_dict(), reg)
|
||||
row["config_xml"] = camp.config_xml
|
||||
try:
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
row["batch_path"] = str(batch_dir)
|
||||
row["image_count"] = len(_iter_batch_images(batch_dir))
|
||||
except Exception as e:
|
||||
row["batch_path"] = None
|
||||
row["image_count"] = 0
|
||||
row["batch_error"] = str(e)
|
||||
return row
|
||||
|
||||
|
||||
def campaign_tasks(
|
||||
campaign_id: str,
|
||||
*,
|
||||
offset: int = 0,
|
||||
limit: int = 50,
|
||||
user: User | None = None,
|
||||
assignee: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
images = _iter_batch_images(batch_dir)
|
||||
from as_platform.labeling.progress import get_assigned_task_ids, user_is_coordinator
|
||||
|
||||
filter_ids: set[str] | None = None
|
||||
if assignee == "me" and user:
|
||||
filter_ids = get_assigned_task_ids(campaign_id, user.id)
|
||||
if not filter_ids and not user_is_coordinator(user):
|
||||
return {
|
||||
"tasks": [],
|
||||
"total": 0,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"hint": "暂无分配给您的任务,请联系协调员在送标工作台均分任务",
|
||||
}
|
||||
|
||||
if filter_ids is not None:
|
||||
filtered = [img for img in images if _task_id_for_image(img, batch_dir) in filter_ids]
|
||||
images = filtered
|
||||
|
||||
total = len(images)
|
||||
slice_imgs = images[offset : offset + limit]
|
||||
tasks: list[dict[str, Any]] = []
|
||||
for img in slice_imgs:
|
||||
tid = _task_id_for_image(img, batch_dir)
|
||||
try:
|
||||
rel = img.relative_to(batch_dir).as_posix()
|
||||
except ValueError:
|
||||
rel = img.name
|
||||
media_path = quote(rel, safe="/")
|
||||
tasks.append(
|
||||
{
|
||||
"id": tid,
|
||||
"data": {
|
||||
"image": f"/api/v1/labeling/media/{campaign_id}/{media_path}",
|
||||
},
|
||||
"meta": {"filename": img.name, "relative_path": rel},
|
||||
}
|
||||
)
|
||||
out: dict[str, Any] = {"tasks": tasks, "total": total, "offset": offset, "limit": limit}
|
||||
if filter_ids is not None and user and assignee == "me":
|
||||
out["my_assigned"] = len(filter_ids)
|
||||
return out
|
||||
|
||||
|
||||
def resolve_media_file(campaign_id: str, rel_path: str) -> Path:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
clean = Path(rel_path)
|
||||
if clean.is_absolute() or ".." in clean.parts:
|
||||
raise PermissionError("invalid path")
|
||||
target = (batch_dir / clean).resolve()
|
||||
if not target.is_file() or not target.is_relative_to(batch_dir.resolve()):
|
||||
raise FileNotFoundError("media not found")
|
||||
return target
|
||||
|
||||
|
||||
def get_annotation(campaign_id: str, task_id: str) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
path = _annotations_dir(batch_dir) / f"{task_id}.json"
|
||||
if not path.is_file():
|
||||
return {"task_id": task_id, "result": None, "annotations": []}
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
return data
|
||||
|
||||
|
||||
def save_annotation(
|
||||
campaign_id: str,
|
||||
task_id: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
user: User | None = None,
|
||||
) -> dict[str, Any]:
|
||||
from as_platform.labeling.progress import assert_can_save_task, mark_task_completed
|
||||
|
||||
if user:
|
||||
assert_can_save_task(campaign_id, task_id, user)
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
path = _annotations_dir(batch_dir) / f"{task_id}.json"
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
extra: dict[str, Any] = {}
|
||||
if user:
|
||||
extra["completed_by_user_id"] = user.id
|
||||
extra["completed_at"] = now
|
||||
out = {"task_id": task_id, **payload, **extra}
|
||||
path.write_text(json.dumps(out, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
if user and _annotation_has_result(path):
|
||||
mark_task_completed(campaign_id, task_id, user.id)
|
||||
return {"ok": True, "path": str(path)}
|
||||
|
||||
|
||||
def _annotation_has_result(path: Path) -> bool:
|
||||
if not path.is_file():
|
||||
return False
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return False
|
||||
result = data.get("result")
|
||||
if result is None:
|
||||
return False
|
||||
if isinstance(result, list):
|
||||
return len(result) > 0
|
||||
if isinstance(result, dict):
|
||||
return len(result) > 0
|
||||
return bool(result)
|
||||
81
platform/as_platform/labeling/batch_stage.py
Normal file
81
platform/as_platform/labeling/batch_stage.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""同步 inbox/sources 批次 batch.meta.yaml 的 stage,与 Campaign 状态一致。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from as_platform.data.batch import read_meta, write_meta
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign
|
||||
from as_platform.labeling.annotate import resolve_campaign_batch_dir
|
||||
|
||||
|
||||
def batch_has_yolo_labels(batch_dir: Path) -> bool:
|
||||
"""批次是否已有导出的 YOLO txt(labels/train 或 labels 根目录)。"""
|
||||
for sub in ("labels/train", "labels"):
|
||||
d = batch_dir / sub
|
||||
if d.is_dir() and any(d.glob("*.txt")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def batch_has_lane_labels(batch_dir: Path) -> bool:
|
||||
"""批次是否已有 UFLD mask 清单(list/train_gt.txt + annotations/*.png)。"""
|
||||
list_path = batch_dir / "list" / "train_gt.txt"
|
||||
if not list_path.is_file():
|
||||
return False
|
||||
ann_dir = batch_dir / "annotations"
|
||||
if not ann_dir.is_dir():
|
||||
return False
|
||||
return any(ann_dir.rglob("*.png"))
|
||||
|
||||
|
||||
def update_campaign_batch_meta_stage(camp: LabelingCampaign, stage: str) -> bool:
|
||||
try:
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
except Exception:
|
||||
return False
|
||||
if not batch_dir.is_dir():
|
||||
return False
|
||||
meta = read_meta(batch_dir) or {}
|
||||
meta["stage"] = stage
|
||||
meta.setdefault("project", camp.project)
|
||||
meta.setdefault("task", camp.task)
|
||||
meta.setdefault("batch", camp.batch)
|
||||
meta.setdefault("location", camp.location or "inbox")
|
||||
if camp.mode:
|
||||
meta.setdefault("mode", camp.mode)
|
||||
write_meta(batch_dir, meta)
|
||||
return True
|
||||
|
||||
|
||||
def update_campaign_batch_meta_stage_by_id(campaign_id: str, stage: str) -> bool:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
return False
|
||||
return update_campaign_batch_meta_stage(camp, stage)
|
||||
|
||||
|
||||
def on_labeling_export_job_succeeded(job: dict) -> None:
|
||||
"""导出 Job 成功且批次已有训练标签时进入 returned(待入库)。"""
|
||||
if job.get("action") != "labeling_export":
|
||||
return
|
||||
params = job.get("params") or {}
|
||||
cid = params.get("campaign_id")
|
||||
if not cid:
|
||||
return
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, str(cid))
|
||||
if not camp:
|
||||
return
|
||||
try:
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
except Exception:
|
||||
return
|
||||
has_labels = (
|
||||
batch_has_lane_labels(batch_dir)
|
||||
if camp.project == "lane"
|
||||
else batch_has_yolo_labels(batch_dir)
|
||||
)
|
||||
if has_labels:
|
||||
update_campaign_batch_meta_stage_by_id(str(cid), "returned")
|
||||
94
platform/as_platform/labeling/lock.py
Normal file
94
platform/as_platform/labeling/lock.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""标注任务 Redis 互斥锁(campaign + task 粒度)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from as_platform.redis.bus import get_redis
|
||||
|
||||
LOCK_TTL_SEC = 300
|
||||
_LOCK_PREFIX = "labeling:lock:"
|
||||
# API 进程内回退(无 Redis 时)
|
||||
_memory: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
def _key(campaign_id: str, task_id: str) -> str:
|
||||
return f"{_LOCK_PREFIX}{campaign_id}:{task_id}"
|
||||
|
||||
|
||||
def _parse_holder(raw: str | None) -> dict[str, Any] | None:
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return {"name": raw}
|
||||
|
||||
|
||||
def acquire_lock(campaign_id: str, task_id: str, *, user_id: int, user_name: str) -> dict[str, Any]:
|
||||
payload = json.dumps({"user_id": user_id, "name": user_name}, ensure_ascii=False)
|
||||
r = get_redis()
|
||||
if not r:
|
||||
now = time.time()
|
||||
mem = _memory.get(_key(campaign_id, task_id))
|
||||
if mem and mem.get("user_id") != user_id and now < mem.get("expires_at", 0):
|
||||
return {"ok": False, "holder": mem.get("name"), "user_id": mem.get("user_id")}
|
||||
_memory[_key(campaign_id, task_id)] = {
|
||||
"user_id": user_id,
|
||||
"name": user_name,
|
||||
"expires_at": now + LOCK_TTL_SEC,
|
||||
}
|
||||
return {"ok": True, "holder": user_name, "ttl_sec": LOCK_TTL_SEC, "backend": "memory"}
|
||||
|
||||
key = _key(campaign_id, task_id)
|
||||
if r.set(key, payload, nx=True, ex=LOCK_TTL_SEC):
|
||||
return {"ok": True, "holder": user_name, "ttl_sec": LOCK_TTL_SEC, "backend": "redis"}
|
||||
existing = _parse_holder(r.get(key))
|
||||
if existing and existing.get("user_id") == user_id:
|
||||
r.expire(key, LOCK_TTL_SEC)
|
||||
return {"ok": True, "holder": user_name, "ttl_sec": LOCK_TTL_SEC, "renewed": True, "backend": "redis"}
|
||||
return {
|
||||
"ok": False,
|
||||
"holder": (existing or {}).get("name"),
|
||||
"user_id": (existing or {}).get("user_id"),
|
||||
"backend": "redis",
|
||||
}
|
||||
|
||||
|
||||
def release_lock(campaign_id: str, task_id: str, *, user_id: int) -> dict[str, Any]:
|
||||
r = get_redis()
|
||||
if not r:
|
||||
key = _key(campaign_id, task_id)
|
||||
mem = _memory.get(key)
|
||||
if mem and mem.get("user_id") == user_id:
|
||||
_memory.pop(key, None)
|
||||
return {"ok": True, "released": True, "backend": "memory"}
|
||||
return {"ok": True, "released": False, "backend": "memory"}
|
||||
|
||||
key = _key(campaign_id, task_id)
|
||||
existing = _parse_holder(r.get(key))
|
||||
if not existing:
|
||||
return {"ok": True, "released": False, "backend": "redis"}
|
||||
if existing.get("user_id") != user_id:
|
||||
return {"ok": False, "holder": existing.get("name"), "backend": "redis"}
|
||||
r.delete(key)
|
||||
return {"ok": True, "released": True, "backend": "redis"}
|
||||
|
||||
|
||||
def renew_lock(campaign_id: str, task_id: str, *, user_id: int) -> dict[str, Any]:
|
||||
r = get_redis()
|
||||
if not r:
|
||||
key = _key(campaign_id, task_id)
|
||||
mem = _memory.get(key)
|
||||
if mem and mem.get("user_id") == user_id:
|
||||
mem["expires_at"] = time.time() + LOCK_TTL_SEC
|
||||
return {"ok": True, "ttl_sec": LOCK_TTL_SEC, "backend": "memory"}
|
||||
return {"ok": False, "backend": "memory"}
|
||||
|
||||
key = _key(campaign_id, task_id)
|
||||
existing = _parse_holder(r.get(key))
|
||||
if not existing or existing.get("user_id") != user_id:
|
||||
return {"ok": False, "holder": (existing or {}).get("name"), "backend": "redis"}
|
||||
r.expire(key, LOCK_TTL_SEC)
|
||||
return {"ok": True, "ttl_sec": LOCK_TTL_SEC, "backend": "redis"}
|
||||
313
platform/as_platform/labeling/progress.py
Normal file
313
platform/as_platform/labeling/progress.py
Normal file
@@ -0,0 +1,313 @@
|
||||
"""标注进度统计与按人分包。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign, LabelingTaskAssignment, User
|
||||
from as_platform.labeling.annotate import (
|
||||
_annotations_dir,
|
||||
_iter_batch_images,
|
||||
_task_id_for_image,
|
||||
resolve_campaign_batch_dir,
|
||||
)
|
||||
|
||||
COORDINATOR_ROLES = frozenset({"labeler", "admin", "engineer", "reviewer"})
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def user_role_codes(user: User) -> set[str]:
|
||||
return {r.code for r in (user.roles or [])}
|
||||
|
||||
|
||||
def user_is_coordinator(user: User) -> bool:
|
||||
codes = user_role_codes(user)
|
||||
if codes & COORDINATOR_ROLES:
|
||||
return True
|
||||
perms: set[str] = set()
|
||||
for r in user.roles or []:
|
||||
for p in r.permissions or []:
|
||||
if p.code:
|
||||
perms.add(p.code)
|
||||
return "*" in perms or "write:labeling_assign" in perms
|
||||
|
||||
|
||||
def list_campaign_task_ids(campaign_id: str) -> list[str]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
images = _iter_batch_images(batch_dir)
|
||||
return [_task_id_for_image(img, batch_dir) for img in images]
|
||||
|
||||
|
||||
def _annotation_has_result(path) -> bool:
|
||||
if not path.is_file():
|
||||
return False
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return False
|
||||
result = data.get("result")
|
||||
if result is None:
|
||||
return False
|
||||
if isinstance(result, list):
|
||||
return len(result) > 0
|
||||
if isinstance(result, dict):
|
||||
return len(result) > 0
|
||||
return bool(result)
|
||||
|
||||
|
||||
def count_completed_tasks(batch_dir) -> set[str]:
|
||||
ann_dir = _annotations_dir(batch_dir)
|
||||
done: set[str] = set()
|
||||
if not ann_dir.is_dir():
|
||||
return done
|
||||
for p in ann_dir.glob("*.json"):
|
||||
if _annotation_has_result(p):
|
||||
done.add(p.stem)
|
||||
return done
|
||||
|
||||
|
||||
def _user_name_map(db, user_ids: set[int]) -> dict[int, str]:
|
||||
if not user_ids:
|
||||
return {}
|
||||
rows = db.query(User).filter(User.id.in_(user_ids)).all()
|
||||
return {u.id: u.name or f"user-{u.id}" for u in rows}
|
||||
|
||||
|
||||
def campaign_progress(campaign_id: str) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
all_ids = [_task_id_for_image(img, batch_dir) for img in _iter_batch_images(batch_dir)]
|
||||
completed_ids = count_completed_tasks(batch_dir)
|
||||
rows = (
|
||||
db.query(LabelingTaskAssignment)
|
||||
.filter(LabelingTaskAssignment.campaign_id == campaign_id)
|
||||
.all()
|
||||
)
|
||||
user_ids = {r.user_id for r in rows} | {r.completed_by_user_id for r in rows if r.completed_by_user_id}
|
||||
names = _user_name_map(db, user_ids)
|
||||
|
||||
total = len(all_ids)
|
||||
completed = len(completed_ids)
|
||||
assigned = len(rows)
|
||||
by_user_agg: dict[int, dict[str, int]] = defaultdict(lambda: {"assigned": 0, "completed": 0})
|
||||
for r in rows:
|
||||
by_user_agg[r.user_id]["assigned"] += 1
|
||||
if r.completed_at or r.task_id in completed_ids:
|
||||
by_user_agg[r.user_id]["completed"] += 1
|
||||
|
||||
by_user = []
|
||||
for uid, stats in sorted(by_user_agg.items(), key=lambda x: names.get(x[0], "")):
|
||||
a = stats["assigned"]
|
||||
c = stats["completed"]
|
||||
by_user.append(
|
||||
{
|
||||
"user_id": uid,
|
||||
"name": names.get(uid, f"user-{uid}"),
|
||||
"assigned": a,
|
||||
"completed": c,
|
||||
"percent": round(100.0 * c / a, 1) if a else 0.0,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"campaign_id": campaign_id,
|
||||
"total_tasks": total,
|
||||
"completed_tasks": completed,
|
||||
"assigned_tasks": assigned,
|
||||
"unassigned_tasks": max(0, total - assigned),
|
||||
"percent": round(100.0 * completed / total, 1) if total else 0.0,
|
||||
"by_user": by_user,
|
||||
}
|
||||
|
||||
|
||||
def campaign_progress_summary(campaign_id: str) -> dict[str, int]:
|
||||
try:
|
||||
p = campaign_progress(campaign_id)
|
||||
return {
|
||||
"total_tasks": p["total_tasks"],
|
||||
"completed_tasks": p["completed_tasks"],
|
||||
"assigned_tasks": p["assigned_tasks"],
|
||||
}
|
||||
except FileNotFoundError:
|
||||
return {"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0}
|
||||
|
||||
|
||||
def get_assigned_task_ids(campaign_id: str, user_id: int | None = None) -> set[str]:
|
||||
with session_scope() as db:
|
||||
q = db.query(LabelingTaskAssignment.task_id).filter(
|
||||
LabelingTaskAssignment.campaign_id == campaign_id
|
||||
)
|
||||
if user_id is not None:
|
||||
q = q.filter(LabelingTaskAssignment.user_id == user_id)
|
||||
return {row[0] for row in q.all()}
|
||||
|
||||
|
||||
def _assign_result(campaign_id: str, created: int) -> dict[str, Any]:
|
||||
prog = campaign_progress(campaign_id)
|
||||
return {"assigned": created, "by_user": prog["by_user"], "progress": prog}
|
||||
|
||||
|
||||
def assign_tasks_even(
|
||||
campaign_id: str,
|
||||
user_ids: list[int],
|
||||
*,
|
||||
assigned_by_user_id: int,
|
||||
) -> dict[str, Any]:
|
||||
if not user_ids:
|
||||
raise ValueError("user_ids 不能为空")
|
||||
now = _utcnow()
|
||||
all_ids = list_campaign_task_ids(campaign_id)
|
||||
with session_scope() as db:
|
||||
existing = {
|
||||
r.task_id
|
||||
for r in db.query(LabelingTaskAssignment)
|
||||
.filter(LabelingTaskAssignment.campaign_id == campaign_id)
|
||||
.all()
|
||||
}
|
||||
unassigned = [tid for tid in all_ids if tid not in existing]
|
||||
users = db.query(User).filter(User.id.in_(user_ids)).all()
|
||||
if len(users) != len(set(user_ids)):
|
||||
raise ValueError("存在无效 user_id")
|
||||
created = 0
|
||||
for i, tid in enumerate(unassigned):
|
||||
uid = user_ids[i % len(user_ids)]
|
||||
db.add(
|
||||
LabelingTaskAssignment(
|
||||
campaign_id=campaign_id,
|
||||
task_id=tid,
|
||||
user_id=uid,
|
||||
assigned_by_user_id=assigned_by_user_id,
|
||||
assigned_at=now,
|
||||
)
|
||||
)
|
||||
created += 1
|
||||
db.flush()
|
||||
return _assign_result(campaign_id, created)
|
||||
|
||||
|
||||
def assign_tasks_explicit(
|
||||
campaign_id: str,
|
||||
items: list[dict[str, Any]],
|
||||
*,
|
||||
assigned_by_user_id: int,
|
||||
) -> dict[str, Any]:
|
||||
now = _utcnow()
|
||||
all_set = set(list_campaign_task_ids(campaign_id))
|
||||
created = 0
|
||||
with session_scope() as db:
|
||||
existing = {
|
||||
r.task_id
|
||||
for r in db.query(LabelingTaskAssignment)
|
||||
.filter(LabelingTaskAssignment.campaign_id == campaign_id)
|
||||
.all()
|
||||
}
|
||||
for item in items:
|
||||
uid = int(item["user_id"])
|
||||
user = db.get(User, uid)
|
||||
if not user:
|
||||
raise ValueError(f"用户不存在: {uid}")
|
||||
for tid in item.get("task_ids") or []:
|
||||
if tid not in all_set:
|
||||
raise ValueError(f"无效 task_id: {tid}")
|
||||
if tid in existing:
|
||||
continue
|
||||
db.add(
|
||||
LabelingTaskAssignment(
|
||||
campaign_id=campaign_id,
|
||||
task_id=tid,
|
||||
user_id=uid,
|
||||
assigned_by_user_id=assigned_by_user_id,
|
||||
assigned_at=now,
|
||||
)
|
||||
)
|
||||
existing.add(tid)
|
||||
created += 1
|
||||
db.flush()
|
||||
return _assign_result(campaign_id, created)
|
||||
|
||||
|
||||
def reassign_task(campaign_id: str, task_id: str, user_id: int) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
row = (
|
||||
db.query(LabelingTaskAssignment)
|
||||
.filter(
|
||||
LabelingTaskAssignment.campaign_id == campaign_id,
|
||||
LabelingTaskAssignment.task_id == task_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not row:
|
||||
raise FileNotFoundError("assignment not found")
|
||||
user = db.get(User, user_id)
|
||||
if not user:
|
||||
raise ValueError(f"用户不存在: {user_id}")
|
||||
row.user_id = user_id
|
||||
db.flush()
|
||||
return row.to_dict()
|
||||
|
||||
|
||||
def release_task_assignment(campaign_id: str, task_id: str) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
row = (
|
||||
db.query(LabelingTaskAssignment)
|
||||
.filter(
|
||||
LabelingTaskAssignment.campaign_id == campaign_id,
|
||||
LabelingTaskAssignment.task_id == task_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not row:
|
||||
raise FileNotFoundError("assignment not found")
|
||||
db.delete(row)
|
||||
db.flush()
|
||||
return {"ok": True, "released": task_id}
|
||||
|
||||
|
||||
def assert_can_save_task(campaign_id: str, task_id: str, user: User) -> None:
|
||||
if user_is_coordinator(user):
|
||||
return
|
||||
codes = user_role_codes(user)
|
||||
if "vendor_labeler" in codes:
|
||||
return
|
||||
with session_scope() as db:
|
||||
row = (
|
||||
db.query(LabelingTaskAssignment)
|
||||
.filter(
|
||||
LabelingTaskAssignment.campaign_id == campaign_id,
|
||||
LabelingTaskAssignment.task_id == task_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not row:
|
||||
raise PermissionError("该图未分配给您,请联系协调员分包")
|
||||
if row.user_id != user.id:
|
||||
raise PermissionError("该图已分配给其他标注员")
|
||||
|
||||
|
||||
def mark_task_completed(campaign_id: str, task_id: str, user_id: int) -> None:
|
||||
now = _utcnow()
|
||||
with session_scope() as db:
|
||||
row = (
|
||||
db.query(LabelingTaskAssignment)
|
||||
.filter(
|
||||
LabelingTaskAssignment.campaign_id == campaign_id,
|
||||
LabelingTaskAssignment.task_id == task_id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if row and not row.completed_at:
|
||||
row.completed_at = now
|
||||
row.completed_by_user_id = user_id
|
||||
86
platform/as_platform/labeling/scope.py
Normal file
86
platform/as_platform/labeling/scope.py
Normal file
@@ -0,0 +1,86 @@
|
||||
"""CatalogScope 与 registry 对齐。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
|
||||
DOMAIN_LABELS = {"dms": "舱内 DMS", "forward": "前向 ADAS"}
|
||||
|
||||
|
||||
def format_scope_key(project: str, task: str, mode: str | None = None) -> str:
|
||||
if project == "lane":
|
||||
return f"lane:{task}"
|
||||
if mode:
|
||||
return f"dms:{task}:{mode}"
|
||||
return f"dms:{task}"
|
||||
|
||||
|
||||
def _dms_registry_api():
|
||||
import sys
|
||||
from pathlib import Path
|
||||
p = WORKSPACE / "datasets" / "dms" / "scripts"
|
||||
if str(p) not in sys.path:
|
||||
sys.path.insert(0, str(p))
|
||||
from task_registry import get_mode_config, resolve_task_id, train_yaml_key
|
||||
return get_mode_config, resolve_task_id, train_yaml_key
|
||||
|
||||
|
||||
def load_dms_registry() -> dict:
|
||||
path = WORKSPACE / "datasets" / "dms" / "datasets.registry.yaml"
|
||||
return yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def load_labeling_registry() -> dict[str, Any]:
|
||||
path = WORKSPACE / "datasets" / "labeling.registry.yaml"
|
||||
if not path.is_file():
|
||||
return {"profiles": {}}
|
||||
return yaml.safe_load(path.read_text(encoding="utf-8")) or {"profiles": {}}
|
||||
|
||||
|
||||
def labeling_profile_key(project: str, task: str, mode: str | None, reg: dict | None = None) -> str:
|
||||
if project == "lane":
|
||||
return f"lane__{task}"
|
||||
get_mode_config, resolve_task_id, train_yaml_key = _dms_registry_api()
|
||||
reg = reg or load_dms_registry()
|
||||
task, mode = resolve_task_id(task, mode)
|
||||
return train_yaml_key(task, mode, reg)
|
||||
|
||||
|
||||
def enrich_batch_labels(batch: dict[str, Any], reg: dict | None = None) -> dict[str, Any]:
|
||||
project = batch.get("project") or "dms"
|
||||
task = batch.get("task") or ""
|
||||
mode = batch.get("mode")
|
||||
out = dict(batch)
|
||||
out["scope_key"] = format_scope_key(project, task, mode)
|
||||
if project == "dms":
|
||||
reg = reg or load_dms_registry()
|
||||
get_mode_config, resolve_task_id, _ = _dms_registry_api()
|
||||
try:
|
||||
task_r, mode_r = resolve_task_id(task, mode)
|
||||
mcfg = get_mode_config(task_r, mode_r, reg)
|
||||
domain = mcfg.get("domain") or "dms"
|
||||
out["domain"] = domain
|
||||
out["domain_label"] = DOMAIN_LABELS.get(domain, domain)
|
||||
out["task_label"] = mcfg.get("label") or task
|
||||
if mode_r:
|
||||
modes = (reg.get("tasks") or {}).get(task_r, {}).get("modes") or {}
|
||||
out["mode_label"] = (modes.get(mode_r) or {}).get("label") or mode_r
|
||||
except Exception:
|
||||
out["domain"] = "dms"
|
||||
out["domain_label"] = DOMAIN_LABELS["dms"]
|
||||
else:
|
||||
out["domain_label"] = "车道线 Lane"
|
||||
out["task_label"] = task
|
||||
try:
|
||||
pk = labeling_profile_key(project, task, mode, reg if project == "dms" else None)
|
||||
out["labeling_profile"] = pk
|
||||
prof = (load_labeling_registry().get("profiles") or {}).get(pk)
|
||||
if prof:
|
||||
out["export_default"] = prof.get("export_default")
|
||||
out["ml_adapter"] = prof.get("ml_adapter")
|
||||
except Exception:
|
||||
out["labeling_profile"] = None
|
||||
return out
|
||||
401
platform/as_platform/labeling/service.py
Normal file
401
platform/as_platform/labeling/service.py
Normal file
@@ -0,0 +1,401 @@
|
||||
"""Campaign 与 pending 批次合并列表。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
from as_platform.data.core import get_pending_report, load_wf
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign, LabelingExportJob, User
|
||||
from as_platform.jobs.queue import enqueue_job, get_job
|
||||
from as_platform.labeling.annotate import resolve_editor_xml, sync_campaign_config_xml
|
||||
from as_platform.labeling.batch_stage import (
|
||||
on_labeling_export_job_succeeded,
|
||||
update_campaign_batch_meta_stage,
|
||||
)
|
||||
from as_platform.labeling.scope import (
|
||||
enrich_batch_labels,
|
||||
format_scope_key,
|
||||
load_dms_registry,
|
||||
load_labeling_registry,
|
||||
)
|
||||
|
||||
|
||||
def _campaign_id(project: str, task: str, mode: str | None, batch: str, location: str) -> str:
|
||||
sk = format_scope_key(project, task, mode)
|
||||
raw = f"{sk}:{batch}:{location}"
|
||||
return hashlib.sha256(raw.encode()).hexdigest()[:20]
|
||||
|
||||
|
||||
def _parse_scope_key(scope_key: str) -> tuple[str, str, str | None]:
|
||||
parts = scope_key.split(":")
|
||||
if parts[0] == "lane":
|
||||
return "lane", parts[1] if len(parts) > 1 else "lane_v1", None
|
||||
if len(parts) >= 3:
|
||||
return "dms", parts[1], parts[2]
|
||||
if len(parts) == 2:
|
||||
return "dms", parts[1], None
|
||||
return "dms", parts[-1], None
|
||||
|
||||
|
||||
def _registry_fallback_batches(wf: dict, reg: dict) -> list[dict[str, Any]]:
|
||||
"""labeling.registry 中有配置但 pending 未扫到的批次(如空 inbox)。"""
|
||||
from pathlib import Path
|
||||
|
||||
from as_platform.data.batch import enrich_batch
|
||||
from as_platform.data.core import proj_root
|
||||
|
||||
profiles = load_labeling_registry().get("profiles") or {}
|
||||
rows: list[dict[str, Any]] = []
|
||||
dms_root = proj_root(wf, "dms")
|
||||
for _pk, prof in profiles.items():
|
||||
scope_key = prof.get("scope_key") or ""
|
||||
project, task, mode = _parse_scope_key(scope_key)
|
||||
if project != "dms":
|
||||
continue
|
||||
batch = mode or task
|
||||
batch_dir = None
|
||||
if mode:
|
||||
try:
|
||||
import sys
|
||||
|
||||
scripts = WORKSPACE / "datasets" / "dms" / "scripts"
|
||||
if str(scripts) not in sys.path:
|
||||
sys.path.insert(0, str(scripts))
|
||||
from task_registry import inbox_dir, resolve_task_id
|
||||
|
||||
task_r, mode_r = resolve_task_id(task, mode)
|
||||
batch_dir = inbox_dir(dms_root, task_r, mode_r, reg)
|
||||
except Exception:
|
||||
batch_dir = dms_root / "inbox" / task / mode
|
||||
else:
|
||||
batch_dir = dms_root / "inbox" / task / batch
|
||||
if not isinstance(batch_dir, Path) or not batch_dir.is_dir():
|
||||
row = {
|
||||
"project": project,
|
||||
"task": task,
|
||||
"mode": mode,
|
||||
"batch": batch,
|
||||
"stage": "raw_pool",
|
||||
"location": "inbox",
|
||||
"path": str(batch_dir) if batch_dir else "",
|
||||
"counts": {"images": 0, "labels": 0},
|
||||
"registry_only": True,
|
||||
}
|
||||
else:
|
||||
row = enrich_batch(
|
||||
batch_dir,
|
||||
project=project,
|
||||
task=task,
|
||||
pack=None,
|
||||
batch=batch,
|
||||
location="inbox",
|
||||
)
|
||||
row["mode"] = mode
|
||||
row["scope_key"] = scope_key
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def list_labeling_batches(
|
||||
*,
|
||||
stage: str | None = None,
|
||||
offset: int = 0,
|
||||
limit: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
wf = load_wf()
|
||||
report = get_pending_report(wf)
|
||||
reg = load_dms_registry()
|
||||
items: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
allowed_stages = ("raw_pool", "out_for_labeling", "returned", "labeling_submitted", "in_review", "review_approved", "review_rejected")
|
||||
|
||||
def _append(b: dict[str, Any]) -> None:
|
||||
if stage and b.get("stage") != stage:
|
||||
return
|
||||
if b.get("stage") not in allowed_stages:
|
||||
return
|
||||
row = enrich_batch_labels(b, reg)
|
||||
cid = _campaign_id(
|
||||
row["project"], row.get("task") or "", row.get("mode"), row["batch"], row.get("location") or "inbox"
|
||||
)
|
||||
key = f"{cid}"
|
||||
if key in seen:
|
||||
return
|
||||
seen.add(key)
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, cid)
|
||||
status = camp.status if camp else "not_opened"
|
||||
if camp:
|
||||
row["assigned_to_user_id"] = camp.assigned_to_user_id
|
||||
row["assigned_to_name"] = camp.assigned_to_name
|
||||
row["campaign_id"] = cid
|
||||
row["campaign_status"] = status
|
||||
if camp and status in ("in_progress", "labeling_submitted"):
|
||||
try:
|
||||
from as_platform.labeling.progress import campaign_progress_summary
|
||||
|
||||
row.update(campaign_progress_summary(cid))
|
||||
except Exception:
|
||||
row.update({"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0})
|
||||
items.append(row)
|
||||
|
||||
for b in report.get("batches", []):
|
||||
_append(b)
|
||||
|
||||
for b in _registry_fallback_batches(wf, reg):
|
||||
_append(b)
|
||||
|
||||
total = len(items)
|
||||
page = items[max(0, offset) : max(0, offset) + max(1, limit)]
|
||||
return {
|
||||
"items": page,
|
||||
"total": total,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"updated_at": report.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
def open_campaign(
|
||||
*,
|
||||
project: str,
|
||||
task: str,
|
||||
batch: str,
|
||||
mode: str | None = None,
|
||||
pack: str | None = None,
|
||||
location: str = "inbox",
|
||||
) -> dict[str, Any]:
|
||||
cid = _campaign_id(project, task, mode, batch, location)
|
||||
config_xml = resolve_editor_xml(project, task, mode)
|
||||
now = datetime.now(timezone.utc)
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, cid)
|
||||
if not camp:
|
||||
camp = LabelingCampaign(
|
||||
id=cid,
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
batch=batch,
|
||||
pack=pack,
|
||||
location=location,
|
||||
status="in_progress",
|
||||
config_xml=config_xml,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
db.add(camp)
|
||||
else:
|
||||
camp.status = "in_progress"
|
||||
camp.updated_at = now
|
||||
sync_campaign_config_xml(camp)
|
||||
db.flush()
|
||||
out = camp.to_dict()
|
||||
out["config_xml"] = camp.config_xml
|
||||
update_campaign_batch_meta_stage(camp, "out_for_labeling")
|
||||
reg = load_dms_registry() if project == "dms" else None
|
||||
row = enrich_batch_labels(out, reg)
|
||||
row["stage"] = "out_for_labeling"
|
||||
return row
|
||||
|
||||
|
||||
def get_campaign(campaign_id: str) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
return None
|
||||
row = camp.to_dict()
|
||||
row["config_xml"] = camp.config_xml
|
||||
reg = load_dms_registry() if row.get("project") == "dms" else None
|
||||
return enrich_batch_labels(row, reg)
|
||||
|
||||
|
||||
def _export_job_id() -> str:
|
||||
return f"lej-{uuid.uuid4().hex[:16]}"
|
||||
|
||||
|
||||
def _record_export_job(campaign_id: str, action: str, job: dict[str, Any]) -> dict[str, Any]:
|
||||
ej_id = _export_job_id()
|
||||
job_id = job.get("id")
|
||||
now = datetime.now(timezone.utc)
|
||||
with session_scope() as db:
|
||||
ej = LabelingExportJob(
|
||||
id=ej_id,
|
||||
campaign_id=campaign_id,
|
||||
action=action,
|
||||
job_id=job_id,
|
||||
status=job.get("status") or "queued",
|
||||
created_at=now,
|
||||
)
|
||||
db.add(ej)
|
||||
out = get_export_job(ej_id)
|
||||
return out or {"id": ej_id, "campaign_id": campaign_id, "action": action, "job_id": job_id}
|
||||
|
||||
|
||||
def _sync_export_job_from_queue(ej: LabelingExportJob) -> None:
|
||||
if not ej.job_id:
|
||||
return
|
||||
job = get_job(ej.job_id)
|
||||
if not job:
|
||||
return
|
||||
ej.status = job.get("status") or ej.status
|
||||
if job.get("finished_at"):
|
||||
try:
|
||||
ej.finished_at = datetime.fromisoformat(str(job["finished_at"]).replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
pass
|
||||
if job.get("result") is not None:
|
||||
ej.result_json = json.dumps(job.get("result"), ensure_ascii=False)
|
||||
if ej.action == "labeling_export" and ej.status in ("succeeded", "completed"):
|
||||
on_labeling_export_job_succeeded(
|
||||
{"action": "labeling_export", "params": {"campaign_id": ej.campaign_id}}
|
||||
)
|
||||
|
||||
|
||||
def get_export_job(export_job_id: str) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
ej = db.get(LabelingExportJob, export_job_id)
|
||||
if not ej:
|
||||
return None
|
||||
_sync_export_job_from_queue(ej)
|
||||
db.flush()
|
||||
return ej.to_dict()
|
||||
|
||||
|
||||
def list_campaign_export_jobs(campaign_id: str, *, limit: int = 30) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
rows = (
|
||||
db.query(LabelingExportJob)
|
||||
.filter_by(campaign_id=campaign_id)
|
||||
.filter(LabelingExportJob.action != "labeling_ml_predict")
|
||||
.order_by(LabelingExportJob.created_at.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
for ej in rows:
|
||||
_sync_export_job_from_queue(ej)
|
||||
db.flush()
|
||||
items = [ej.to_dict() for ej in rows]
|
||||
return {"items": items, "campaign_id": campaign_id}
|
||||
|
||||
|
||||
def list_labeling_assignees() -> dict[str, Any]:
|
||||
"""可指派为批次负责人的用户(标注相关角色)。"""
|
||||
role_codes = ("labeler", "internal_labeler", "vendor_labeler", "engineer", "admin")
|
||||
with session_scope() as db:
|
||||
users = (
|
||||
db.query(User)
|
||||
.filter(User.is_active.is_(True))
|
||||
.order_by(User.name)
|
||||
.all()
|
||||
)
|
||||
items = []
|
||||
for u in users:
|
||||
codes = {r.code for r in (u.roles or [])}
|
||||
if codes.intersection(role_codes):
|
||||
items.append({"id": u.id, "name": u.name or f"user-{u.id}", "roles": sorted(codes)})
|
||||
return {"items": items}
|
||||
|
||||
|
||||
def _find_batch_for_campaign_id(campaign_id: str) -> dict[str, Any] | None:
|
||||
"""由确定性 campaign_id 反查 pending / registry 批次行。"""
|
||||
wf = load_wf()
|
||||
reg = load_dms_registry()
|
||||
candidates: list[dict[str, Any]] = []
|
||||
report = get_pending_report(wf)
|
||||
candidates.extend(report.get("batches") or [])
|
||||
candidates.extend(_registry_fallback_batches(wf, reg))
|
||||
for b in candidates:
|
||||
cid = _campaign_id(
|
||||
b.get("project") or "dms",
|
||||
b.get("task") or "",
|
||||
b.get("mode"),
|
||||
b.get("batch") or "",
|
||||
b.get("location") or "inbox",
|
||||
)
|
||||
if cid == campaign_id:
|
||||
return b
|
||||
return None
|
||||
|
||||
|
||||
def ensure_campaign_record(campaign_id: str) -> None:
|
||||
"""提交/导出前保证 DB 中有 LabelingCampaign(未点「进入标注」时自动创建)。"""
|
||||
with session_scope() as db:
|
||||
if db.get(LabelingCampaign, campaign_id):
|
||||
return
|
||||
batch = _find_batch_for_campaign_id(campaign_id)
|
||||
if not batch:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
if batch.get("registry_only"):
|
||||
raise ValueError("该条目为任务模板占位,无真实 inbox 批次目录,请先送标入湖或从「进入标注」开启真实批次")
|
||||
open_campaign(
|
||||
project=batch.get("project") or "dms",
|
||||
task=batch.get("task") or "",
|
||||
batch=batch["batch"],
|
||||
mode=batch.get("mode"),
|
||||
pack=batch.get("pack"),
|
||||
location=batch.get("location") or "inbox",
|
||||
)
|
||||
|
||||
|
||||
def assign_campaign(campaign_id: str, user_id: int | None) -> dict[str, Any]:
|
||||
now = datetime.now(timezone.utc)
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
if user_id is None:
|
||||
camp.assigned_to_user_id = None
|
||||
camp.assigned_to_name = None
|
||||
else:
|
||||
user = db.get(User, user_id)
|
||||
if not user:
|
||||
raise ValueError(f"用户不存在: {user_id}")
|
||||
camp.assigned_to_user_id = user_id
|
||||
camp.assigned_to_name = user.name
|
||||
camp.updated_at = now
|
||||
db.flush()
|
||||
out = camp.to_dict()
|
||||
reg = load_dms_registry() if out.get("project") == "dms" else None
|
||||
return enrich_batch_labels(out, reg)
|
||||
|
||||
|
||||
def submit_campaign(campaign_id: str) -> dict[str, Any]:
|
||||
ensure_campaign_record(campaign_id)
|
||||
now = datetime.now(timezone.utc)
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
camp.status = "in_review"
|
||||
camp.updated_at = now
|
||||
db.flush()
|
||||
out = camp.to_dict()
|
||||
update_campaign_batch_meta_stage(camp, "in_review")
|
||||
reg = load_dms_registry() if out.get("project") == "dms" else None
|
||||
row = enrich_batch_labels(out, reg)
|
||||
row["stage"] = "in_review"
|
||||
return row
|
||||
|
||||
|
||||
def trigger_labeling_export(campaign_id: str) -> dict[str, Any]:
|
||||
row = get_campaign(campaign_id)
|
||||
if not row:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
job = enqueue_job(
|
||||
"labeling_export",
|
||||
{
|
||||
"campaign_id": campaign_id,
|
||||
"export_default": row.get("export_default"),
|
||||
"scope_key": row.get("scope_key"),
|
||||
"batch": row.get("batch"),
|
||||
},
|
||||
async_run=True,
|
||||
)
|
||||
ej = _record_export_job(campaign_id, "labeling_export", job)
|
||||
return {"ok": True, "job": job, "export_job": ej, "export_default": row.get("export_default")}
|
||||
116
platform/as_platform/labeling/vendor_import.py
Normal file
116
platform/as_platform/labeling/vendor_import.py
Normal file
@@ -0,0 +1,116 @@
|
||||
"""第三方标注回传包(ZIP)导入到 Campaign 批次目录。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.labeling.annotate import ANNOTATIONS_DIRNAME, resolve_campaign_batch_dir
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import LabelingCampaign, LabelingCampaignAccess
|
||||
|
||||
|
||||
def grant_campaign_access(
|
||||
db,
|
||||
*,
|
||||
campaign_id: str,
|
||||
principal_type: str = "role",
|
||||
principal_id: str = "vendor_labeler",
|
||||
access_role: str = "vendor",
|
||||
) -> dict:
|
||||
row = (
|
||||
db.query(LabelingCampaignAccess)
|
||||
.filter_by(campaign_id=campaign_id, principal_type=principal_type, principal_id=principal_id)
|
||||
.first()
|
||||
)
|
||||
if not row:
|
||||
row = LabelingCampaignAccess(
|
||||
campaign_id=campaign_id,
|
||||
principal_type=principal_type,
|
||||
principal_id=principal_id,
|
||||
access_role=access_role,
|
||||
)
|
||||
db.add(row)
|
||||
db.flush()
|
||||
return row.to_dict()
|
||||
|
||||
|
||||
def import_vendor_zip(campaign_id: str, raw: bytes) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if not camp:
|
||||
raise FileNotFoundError("campaign not found")
|
||||
batch_dir = resolve_campaign_batch_dir(camp)
|
||||
grant_campaign_access(db, campaign_id=campaign_id)
|
||||
|
||||
batch_dir.mkdir(parents=True, exist_ok=True)
|
||||
images_dir = batch_dir / "images"
|
||||
labels_dir = batch_dir / "labels"
|
||||
images_dir.mkdir(parents=True, exist_ok=True)
|
||||
labels_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
img_count = 0
|
||||
label_count = 0
|
||||
manifest: dict[str, Any] = {}
|
||||
|
||||
with zipfile.ZipFile(BytesIO(raw)) as zf:
|
||||
for info in zf.infolist():
|
||||
if info.is_dir():
|
||||
continue
|
||||
name = info.filename.replace("\\", "/").lstrip("/")
|
||||
lower = name.lower()
|
||||
if lower == "manifest.json":
|
||||
manifest = json.loads(zf.read(info))
|
||||
continue
|
||||
data = zf.read(info)
|
||||
base = Path(name).name
|
||||
if not base:
|
||||
continue
|
||||
if "/images/" in lower or lower.startswith("images/"):
|
||||
dest = images_dir / base
|
||||
dest.write_bytes(data)
|
||||
img_count += 1
|
||||
elif "/labels/" in lower or lower.startswith("labels/"):
|
||||
dest = labels_dir / base
|
||||
dest.write_bytes(data)
|
||||
label_count += 1
|
||||
elif base.endswith((".jpg", ".jpeg", ".png", ".bmp", ".webp")):
|
||||
(images_dir / base).write_bytes(data)
|
||||
img_count += 1
|
||||
elif base.endswith((".txt", ".json")):
|
||||
(labels_dir / base).write_bytes(data)
|
||||
label_count += 1
|
||||
|
||||
ann_dir = batch_dir / ANNOTATIONS_DIRNAME
|
||||
ann_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"campaign_id": campaign_id,
|
||||
"batch_path": str(batch_dir),
|
||||
"images_imported": img_count,
|
||||
"labels_imported": label_count,
|
||||
"manifest": manifest,
|
||||
}
|
||||
|
||||
|
||||
def list_registry_profiles() -> dict[str, Any]:
|
||||
from as_platform.labeling.scope import load_labeling_registry
|
||||
|
||||
reg = load_labeling_registry()
|
||||
profiles = reg.get("profiles") or {}
|
||||
items = []
|
||||
for key, prof in profiles.items():
|
||||
items.append(
|
||||
{
|
||||
"profile_key": key,
|
||||
"editor_template": prof.get("editor_template"),
|
||||
"export_default": prof.get("export_default"),
|
||||
"ml_adapter": prof.get("ml_adapter"),
|
||||
"type": prof.get("type"),
|
||||
}
|
||||
)
|
||||
return {"profiles": items, "version": reg.get("version")}
|
||||
Reference in New Issue
Block a user