feat: HSAP platform v2 — modular navigation, quality review, audit log, world model simulation
Major changes: - New frontend (platform/web/): Vite + React 18 + TypeScript + Tailwind - 4-module navigation: 数据送标 / 模型管理 / 车队管理 / 系统管理 - Data catalog with charts (DMS/ADAS/Lane 3-tab view) - Quality review workflow (标注质检): Good/Fine/Bad scoring with auto-advance - Audit enhancements: batch operations, rejection categories, Feishu notifications - Operation audit log (操作日志) - World model simulation studio (仿真工坊) - Dataset version management with snapshots and diff - ADAS 7-class dataset integration (138K images organized + compressed) - User management with Feishu integration and pagination - CRUD/search/filter on all pages, card layout redesign - PIL-optimized image overlay rendering - Auto-snapshot on build, in_review workflow stage - Removed embedded algorithm code (now in workspace)
This commit is contained in:
@@ -15,6 +15,7 @@ from as_platform.data.lake import (
|
||||
get_candidate,
|
||||
link_candidate_analysis_job,
|
||||
list_candidates,
|
||||
promote_candidate_to_inbox,
|
||||
write_candidate_upload,
|
||||
)
|
||||
|
||||
@@ -34,4 +35,5 @@ __all__ = [
|
||||
"get_candidate",
|
||||
"link_candidate_analysis_job",
|
||||
"analyze_uploaded_candidate",
|
||||
"promote_candidate_to_inbox",
|
||||
]
|
||||
|
||||
@@ -13,7 +13,7 @@ from as_platform.config import WORKSPACE
|
||||
CATALOG_CACHE_FILE = WORKSPACE / "manifests" / "catalog_cache.json"
|
||||
CATALOG_CACHE_TTL_SEC = int(os.environ.get("AS_CATALOG_CACHE_TTL_SEC", "300"))
|
||||
CATALOG_USE_REPORTS = os.environ.get("AS_CATALOG_USE_REPORTS", "1").lower() in ("1", "true", "yes")
|
||||
CATALOG_CACHE_VERSION = 3
|
||||
CATALOG_CACHE_VERSION = 7
|
||||
|
||||
REPORTS_DIR = WORKSPACE / "reports"
|
||||
DMS_SUMMARY_CSV = REPORTS_DIR / "dms_task_image_summary.csv"
|
||||
@@ -111,8 +111,14 @@ def build_catalog_signature(wf: dict, proj_root_fn) -> dict[str, Any]:
|
||||
import yaml
|
||||
|
||||
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
|
||||
for task in (reg.get("tasks") or {}).keys():
|
||||
dirs.append(_dir_fingerprint(root / "inbox" / task))
|
||||
for task, tcfg in (reg.get("tasks") or {}).items():
|
||||
if tcfg.get("type") == "multi":
|
||||
for mcfg in (tcfg.get("modes") or {}).values():
|
||||
rel = mcfg.get("inbox", "")
|
||||
if rel:
|
||||
dirs.append(_dir_fingerprint(root / rel))
|
||||
else:
|
||||
dirs.append(_dir_fingerprint(root / (tcfg.get("inbox") or f"inbox/{task}")))
|
||||
if pname == "lane":
|
||||
dirs.append(_dir_fingerprint(_pack_registry_path("lane", root, wf), scan_children=False))
|
||||
|
||||
@@ -140,13 +146,41 @@ def save_disk_cache(payload: dict[str, Any]) -> None:
|
||||
CATALOG_CACHE_FILE.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
|
||||
def _iter_dms_packs(task_entry: dict[str, Any]):
|
||||
modes = task_entry.get("modes")
|
||||
if isinstance(modes, dict):
|
||||
for mode_entry in modes.values():
|
||||
for pack in mode_entry.get("packs") or []:
|
||||
if isinstance(pack, dict):
|
||||
yield pack, mode_entry.get("type")
|
||||
return
|
||||
for pack in task_entry.get("packs") or []:
|
||||
if isinstance(pack, dict):
|
||||
yield pack, task_entry.get("type")
|
||||
|
||||
|
||||
def _entry_needs_bbox_refresh(entry: dict[str, Any]) -> bool:
|
||||
if entry.get("type") == "classify":
|
||||
return False
|
||||
has_class = bool(entry.get("class_counts"))
|
||||
any_pts = False
|
||||
any_boxes = False
|
||||
for pack in entry.get("packs") or []:
|
||||
if pack.get("bbox_points"):
|
||||
any_pts = True
|
||||
if int(pack.get("total_boxes") or 0) > 0:
|
||||
any_boxes = True
|
||||
return not any_pts and (any_boxes or has_class)
|
||||
|
||||
|
||||
def _catalog_has_empty_bbox(catalog: dict[str, Any]) -> bool:
|
||||
for task in (catalog.get("dms") or {}).values():
|
||||
for pack in task.get("packs") or []:
|
||||
boxes = int(pack.get("total_boxes") or 0)
|
||||
pts = pack.get("bbox_points") or []
|
||||
if boxes > 0 and not pts:
|
||||
return True
|
||||
if task.get("type") == "multi":
|
||||
for mode_entry in (task.get("modes") or {}).values():
|
||||
if _entry_needs_bbox_refresh(mode_entry):
|
||||
return True
|
||||
elif _entry_needs_bbox_refresh(task):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -198,7 +232,8 @@ def store_catalog_cache(signature: dict[str, Any], data: dict[str, Any], *, buil
|
||||
"data": data,
|
||||
}
|
||||
_CATALOG_MEM_CACHE = payload
|
||||
save_disk_cache(payload)
|
||||
if not _catalog_has_empty_bbox(data):
|
||||
save_disk_cache(payload)
|
||||
return payload
|
||||
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from as_platform.config import WORKSPACE, LANE_DATA_VIZ_ENABLED
|
||||
from as_platform.data.batch import META_FILENAME, enrich_batch, write_meta
|
||||
from as_platform.config import WORKSPACE, WORKSPACE_ROOT, LANE_DATA_VIZ_ENABLED
|
||||
from as_platform.data.batch import META_FILENAME, dms_has_images, enrich_batch, write_meta
|
||||
from as_platform.data.catalog_cache import (
|
||||
build_catalog_signature,
|
||||
get_cached_catalog,
|
||||
@@ -20,6 +20,30 @@ from as_platform.data.catalog_cache import (
|
||||
store_catalog_cache,
|
||||
)
|
||||
|
||||
import sys as _sys
|
||||
|
||||
def _dms_scripts_dir() -> Path:
|
||||
for cand in (
|
||||
WORKSPACE / "datasets" / "dms" / "scripts",
|
||||
WORKSPACE / "datasets" / "dms.embedded.bak" / "scripts",
|
||||
):
|
||||
if (cand / "task_registry.py").is_file():
|
||||
return cand
|
||||
return WORKSPACE / "datasets" / "dms" / "scripts"
|
||||
|
||||
|
||||
_DMS_SCRIPTS = _dms_scripts_dir()
|
||||
if str(_DMS_SCRIPTS) not in _sys.path:
|
||||
_sys.path.insert(0, str(_DMS_SCRIPTS))
|
||||
from task_registry import ( # noqa: E402
|
||||
DOMAIN_LABELS,
|
||||
get_mode_config,
|
||||
inbox_dir,
|
||||
iter_catalog_tasks,
|
||||
report_task_key,
|
||||
task_defs_for_pending,
|
||||
)
|
||||
|
||||
MAX_LABEL_FILES_PER_PACK = 2000
|
||||
MAX_BBOX_POINTS_PER_PACK = 1500
|
||||
MAX_LANE_MASK_SAMPLES_PER_PACK = 500
|
||||
@@ -60,8 +84,36 @@ def resolve_pack(project: str, root: Path, wf: dict, name: str) -> str:
|
||||
raise ValueError(f"[{project}] 未知包: {name},已登记: {known}")
|
||||
|
||||
|
||||
def _pack_dir_usable(path: Path) -> bool:
|
||||
try:
|
||||
return path.is_dir() and any(path.iterdir())
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _dms_workspace_pack_dir(pack_name: str) -> Path | None:
|
||||
ws = WORKSPACE_ROOT
|
||||
if ws is None:
|
||||
return None
|
||||
cand = ws / "DMS/DATASET/packs" / pack_name
|
||||
return cand if _pack_dir_usable(cand) else None
|
||||
|
||||
|
||||
def resolve_pack_dir(project: str, root: Path, wf: dict, name: str) -> Path:
|
||||
return (root / resolve_pack(project, root, wf, name)).resolve()
|
||||
rel = resolve_pack(project, root, wf, name)
|
||||
primary = root / rel
|
||||
if _pack_dir_usable(primary):
|
||||
return primary.resolve()
|
||||
reg = load_pack_registry(project, root, wf)
|
||||
pack_name = reg.get("aliases", {}).get(name, name)
|
||||
if project == "dms":
|
||||
fallback = _dms_workspace_pack_dir(pack_name)
|
||||
if fallback is not None:
|
||||
return fallback
|
||||
try:
|
||||
return primary.resolve()
|
||||
except OSError:
|
||||
return primary
|
||||
|
||||
|
||||
def _read_jsonl_tail(path: Path, n: int = 10) -> list[dict]:
|
||||
@@ -111,28 +163,66 @@ def get_pending_report(wf: dict | None = None) -> dict[str, Any]:
|
||||
ingest_log = root / "manifests" / "ingest_log.jsonl"
|
||||
proj["recent_ingest"] = _read_jsonl_tail(ingest_log, 10)
|
||||
|
||||
for task, tcfg in reg.get("tasks", {}).items():
|
||||
proj["task_defs"][task] = {
|
||||
"type": tcfg.get("type"),
|
||||
"nc": tcfg.get("nc"),
|
||||
"names": tcfg.get("names"),
|
||||
"task_dir": tcfg.get("task_dir", task),
|
||||
}
|
||||
inbox_batches: list[str] = []
|
||||
ib = root / "inbox" / task
|
||||
if ib.is_dir():
|
||||
inbox_batches = [
|
||||
d.name for d in ib.iterdir()
|
||||
if d.is_dir() and not d.name.startswith(".")
|
||||
]
|
||||
proj["task_defs"] = task_defs_for_pending(reg)
|
||||
|
||||
sources_pending: dict[str, list[str]] = {}
|
||||
def _scan_task_inbox(task_id: str, mode: str | None) -> list[str]:
|
||||
ib = inbox_dir(root, task_id, mode, reg)
|
||||
if not ib.is_dir():
|
||||
return []
|
||||
subdirs = [
|
||||
d.name for d in ib.iterdir() if d.is_dir() and not d.name.startswith(".")
|
||||
]
|
||||
if subdirs:
|
||||
return subdirs
|
||||
tcfg = reg.get("tasks", {}).get(task_id, {})
|
||||
if tcfg.get("type") == "multi" and mode:
|
||||
if dms_has_images(ib):
|
||||
return [mode]
|
||||
return []
|
||||
|
||||
def _multi_inbox_batch_rows(task: str, mode: str) -> list[dict[str, Any]]:
|
||||
"""multi 任务:inbox 可能即批次根目录(如 inbox/dam/batch_0516)。"""
|
||||
ib = inbox_dir(root, task, mode, reg)
|
||||
if not ib.is_dir():
|
||||
return []
|
||||
rows: list[dict[str, Any]] = []
|
||||
subdirs = [
|
||||
d.name for d in ib.iterdir() if d.is_dir() and not d.name.startswith(".")
|
||||
]
|
||||
if subdirs:
|
||||
for batch_name in subdirs:
|
||||
row = enrich_batch(
|
||||
ib / batch_name,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=None,
|
||||
batch=batch_name,
|
||||
location="inbox",
|
||||
)
|
||||
row["mode"] = mode
|
||||
rows.append(row)
|
||||
elif dms_has_images(ib) or (ib / META_FILENAME).is_file():
|
||||
row = enrich_batch(
|
||||
ib,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=None,
|
||||
batch=mode,
|
||||
location="inbox",
|
||||
)
|
||||
row["mode"] = mode
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
def _scan_sources(task_id: str, mode: str | None) -> dict[str, list[str]]:
|
||||
mcfg = get_mode_config(task_id, mode, reg)
|
||||
pending: dict[str, list[str]] = {}
|
||||
for pack_name in all_names:
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
src_root = pack_dir / tcfg["task_dir"] / src_sub
|
||||
src_root = pack_dir / mcfg["task_dir"] / src_sub
|
||||
if src_root.is_dir():
|
||||
batches = [
|
||||
d.name for d in src_root.iterdir()
|
||||
@@ -141,44 +231,63 @@ def get_pending_report(wf: dict | None = None) -> dict[str, Any]:
|
||||
and not d.name.startswith(".")
|
||||
]
|
||||
if batches:
|
||||
sources_pending[pack_name] = batches
|
||||
pending[pack_name] = batches
|
||||
return pending
|
||||
|
||||
proj["tasks"][task] = {
|
||||
"inbox": inbox_batches,
|
||||
"sources": sources_pending,
|
||||
}
|
||||
|
||||
for batch_name in inbox_batches:
|
||||
batch_dir = ib / batch_name
|
||||
report["batches"].append(
|
||||
enrich_batch(
|
||||
batch_dir,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=None,
|
||||
batch=batch_name,
|
||||
location="inbox",
|
||||
)
|
||||
)
|
||||
|
||||
for pack_name, batch_list in sources_pending.items():
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
src_root = pack_dir / tcfg["task_dir"] / src_sub
|
||||
for batch_name in batch_list:
|
||||
batch_dir = src_root / batch_name
|
||||
for task, tcfg in reg.get("tasks", {}).items():
|
||||
if tcfg.get("type") == "multi":
|
||||
task_inbox: dict[str, list[str]] = {}
|
||||
task_sources: dict[str, dict[str, list[str]]] = {}
|
||||
for mode in (tcfg.get("modes") or {}):
|
||||
task_inbox[mode] = _scan_task_inbox(task, mode)
|
||||
task_sources[mode] = _scan_sources(task, mode)
|
||||
for row in _multi_inbox_batch_rows(task, mode):
|
||||
report["batches"].append(row)
|
||||
for pack_name, batch_list in task_sources[mode].items():
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
src_root = pack_dir / get_mode_config(task, mode, reg)["task_dir"] / src_sub
|
||||
for batch_name in batch_list:
|
||||
row = enrich_batch(
|
||||
src_root / batch_name,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=pack_name,
|
||||
batch=batch_name,
|
||||
location="sources",
|
||||
)
|
||||
row["mode"] = mode
|
||||
report["batches"].append(row)
|
||||
proj["tasks"][task] = {"inbox": task_inbox, "sources": task_sources}
|
||||
else:
|
||||
inbox_batches = _scan_task_inbox(task, None)
|
||||
sources_pending = _scan_sources(task, None)
|
||||
proj["tasks"][task] = {"inbox": inbox_batches, "sources": sources_pending}
|
||||
ib = inbox_dir(root, task, None, reg)
|
||||
for batch_name in inbox_batches:
|
||||
report["batches"].append(
|
||||
enrich_batch(
|
||||
batch_dir,
|
||||
ib / batch_name,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=pack_name,
|
||||
pack=None,
|
||||
batch=batch_name,
|
||||
location="sources",
|
||||
location="inbox",
|
||||
)
|
||||
)
|
||||
for pack_name, batch_list in sources_pending.items():
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
src_root = pack_dir / tcfg["task_dir"] / src_sub
|
||||
for batch_name in batch_list:
|
||||
report["batches"].append(
|
||||
enrich_batch(
|
||||
src_root / batch_name,
|
||||
project="dms",
|
||||
task=task,
|
||||
pack=pack_name,
|
||||
batch=batch_name,
|
||||
location="sources",
|
||||
)
|
||||
)
|
||||
|
||||
if pname == "lane":
|
||||
proj["packs"] = {}
|
||||
@@ -297,7 +406,7 @@ def _count_images_in_dir(img_dir: Path) -> int:
|
||||
try:
|
||||
with os.scandir(img_dir) as it:
|
||||
for entry in it:
|
||||
if not entry.is_file(follow_symlinks=False):
|
||||
if not entry.is_file(follow_symlinks=True):
|
||||
continue
|
||||
if Path(entry.name).suffix.lower() in IMAGE_EXTS:
|
||||
total += 1
|
||||
@@ -325,8 +434,8 @@ def _iter_label_files(label_dirs: list[Path]):
|
||||
continue
|
||||
try:
|
||||
with os.scandir(label_dir) as it:
|
||||
stack = [entry.path for entry in it if entry.is_dir(follow_symlinks=False)]
|
||||
files = [entry.path for entry in it if entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt")]
|
||||
stack = [entry.path for entry in it if entry.is_dir(follow_symlinks=True)]
|
||||
files = [entry.path for entry in it if entry.is_file(follow_symlinks=True) and entry.name.endswith(".txt")]
|
||||
except OSError:
|
||||
continue
|
||||
for fp in files:
|
||||
@@ -336,9 +445,9 @@ def _iter_label_files(label_dirs: list[Path]):
|
||||
try:
|
||||
with os.scandir(current) as it:
|
||||
for entry in it:
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
if entry.is_dir(follow_symlinks=True):
|
||||
stack.append(entry.path)
|
||||
elif entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt"):
|
||||
elif entry.is_file(follow_symlinks=True) and entry.name.endswith(".txt"):
|
||||
yield Path(entry.path)
|
||||
except OSError:
|
||||
continue
|
||||
@@ -348,6 +457,49 @@ def _label_dirs_for_task(task_data: Path) -> list[Path]:
|
||||
return [task_data / "labels" / "train", task_data / "labels" / "val", task_data / "labels"]
|
||||
|
||||
|
||||
def _task_data_has_labels(task_data: Path) -> bool:
|
||||
return any(p.is_dir() for p in _label_dirs_for_task(task_data))
|
||||
|
||||
|
||||
def resolve_task_data_for_scan(pack_dir: Path, task_dir: str) -> Path:
|
||||
"""解析可扫描的 task 数据目录。
|
||||
|
||||
DAM 等批次在宿主机上常为指向绝对路径的 symlink,Docker 内 resolve() 会落到
|
||||
容器外路径导致 is_dir() 为 False、bbox 采样为空。此处回映射到 AS_WORKSPACE_ROOT
|
||||
并尝试 pack 级 _dam_stash_* 目录。
|
||||
"""
|
||||
task_data = pack_dir / task_dir
|
||||
if _task_data_has_labels(task_data):
|
||||
return task_data
|
||||
|
||||
if task_data.is_symlink():
|
||||
try:
|
||||
raw = os.readlink(task_data)
|
||||
link = Path(raw)
|
||||
candidates: list[Path] = []
|
||||
if link.is_absolute():
|
||||
s = str(link).replace("\\", "/")
|
||||
for marker in ("DMS/DATASET/", "DMS/DATASET"):
|
||||
idx = s.find(marker)
|
||||
if idx >= 0 and WORKSPACE_ROOT:
|
||||
candidates.append(WORKSPACE_ROOT / s[idx:])
|
||||
else:
|
||||
candidates.append((task_data.parent / link).resolve())
|
||||
for cand in candidates:
|
||||
if _task_data_has_labels(cand):
|
||||
return cand
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
leaf = Path(task_dir).name
|
||||
if leaf.startswith("batch_"):
|
||||
stash = pack_dir / f"_dam_stash_{leaf.split('_', 1)[1]}"
|
||||
if _task_data_has_labels(stash):
|
||||
return stash
|
||||
|
||||
return task_data
|
||||
|
||||
|
||||
def _parse_bbox_wh(parts: list[str]) -> list[float] | None:
|
||||
if len(parts) < 5:
|
||||
return None
|
||||
@@ -574,16 +726,80 @@ def _catalog_signature(wf: dict) -> dict[str, Any]:
|
||||
return build_catalog_signature(wf, proj_root)
|
||||
|
||||
|
||||
def _merge_multi_task_entry(
|
||||
dms: dict[str, Any],
|
||||
task_id: str,
|
||||
tcfg: dict,
|
||||
legacy_modes: dict[str, dict],
|
||||
) -> None:
|
||||
entry = dms.setdefault(task_id, {})
|
||||
entry["type"] = "multi"
|
||||
entry["domain"] = tcfg.get("domain", "dms")
|
||||
entry["label"] = tcfg.get("label", task_id)
|
||||
entry["domain_label"] = DOMAIN_LABELS.get(tcfg.get("domain", "dms"), tcfg.get("domain", "dms"))
|
||||
modes = entry.setdefault("modes", {})
|
||||
for mode, mcfg in (tcfg.get("modes") or {}).items():
|
||||
mode_entry = modes.setdefault(mode, {})
|
||||
mode_entry.setdefault("label", mcfg.get("label", mode))
|
||||
mode_entry.setdefault("type", mcfg.get("type", "detect"))
|
||||
mode_entry.setdefault("nc", mcfg.get("nc"))
|
||||
mode_entry.setdefault("names", mcfg.get("names"))
|
||||
mode_entry.setdefault("packs", [])
|
||||
mode_entry.setdefault("class_counts", {})
|
||||
if mode in legacy_modes:
|
||||
leg = legacy_modes[mode]
|
||||
if not mode_entry.get("packs") and leg.get("packs"):
|
||||
mode_entry["packs"] = leg["packs"]
|
||||
if not mode_entry.get("class_counts") and leg.get("class_counts"):
|
||||
mode_entry["class_counts"] = leg["class_counts"]
|
||||
stray = entry.pop("packs", None)
|
||||
first_mode = next(iter(modes), None)
|
||||
if stray and isinstance(stray, list) and first_mode and not modes[first_mode].get("packs"):
|
||||
modes[first_mode]["packs"] = stray
|
||||
entry.pop("nc", None)
|
||||
entry.pop("names", None)
|
||||
|
||||
|
||||
def _normalize_catalog_dms(dms: dict[str, Any], reg: dict) -> dict[str, Any]:
|
||||
"""修复旧缓存:isa/isa_class→forward;dam_0417→dam;补全 multi 结构。"""
|
||||
if not dms:
|
||||
return dms
|
||||
tasks_cfg = (reg or {}).get("tasks") or {}
|
||||
|
||||
legacy_forward = {}
|
||||
for old_key, mode in (("isa", "detect"), ("isa_class", "classify")):
|
||||
if old_key in dms:
|
||||
legacy_forward[mode] = dms.pop(old_key)
|
||||
fwd_cfg = tasks_cfg.get("forward") or {}
|
||||
if fwd_cfg.get("type") == "multi" or legacy_forward:
|
||||
_merge_multi_task_entry(dms, "forward", fwd_cfg, legacy_forward)
|
||||
|
||||
legacy_dam: dict[str, dict] = {}
|
||||
if "dam_0417" in dms:
|
||||
legacy_dam["batch_0417"] = dms.pop("dam_0417")
|
||||
if "dam" in dms and (tasks_cfg.get("dam") or {}).get("type") == "multi":
|
||||
old_dam = dms.pop("dam")
|
||||
if old_dam.get("modes"):
|
||||
dms["dam"] = old_dam
|
||||
else:
|
||||
legacy_dam["batch_0516"] = old_dam
|
||||
dam_cfg = tasks_cfg.get("dam") or {}
|
||||
if dam_cfg.get("type") == "multi" or legacy_dam:
|
||||
_merge_multi_task_entry(dms, "dam", dam_cfg, legacy_dam)
|
||||
|
||||
return dms
|
||||
|
||||
|
||||
def _build_catalog(wf: dict, *, prefer_reports: bool = True) -> tuple[dict[str, Any], str]:
|
||||
out: dict[str, Any] = {"workspace": str(WORKSPACE), "dms": {}, "lane": {}}
|
||||
build_source = "scan"
|
||||
|
||||
reports = load_dms_reports() if prefer_reports else None
|
||||
reports = load_dms_reports()
|
||||
report_splits: dict[tuple[str, str], dict[str, int]] = {}
|
||||
report_classes: dict[str, dict[str, int]] = {}
|
||||
if reports:
|
||||
report_splits, report_classes = reports
|
||||
build_source = "reports"
|
||||
build_source = "reports" if prefer_reports else "scan+reports"
|
||||
|
||||
root = proj_root(wf, "dms")
|
||||
reg_path = root / wf["projects"]["dms"]["registry"]
|
||||
@@ -620,61 +836,91 @@ def _build_catalog(wf: dict, *, prefer_reports: bool = True) -> tuple[dict[str,
|
||||
if summary_path.is_file():
|
||||
class_by_task = _parse_class_summary(summary_path.read_text(encoding="utf-8"))
|
||||
|
||||
for task, tcfg in reg.get("tasks", {}).items():
|
||||
entry: dict[str, Any] = {
|
||||
"type": tcfg.get("type"),
|
||||
"nc": tcfg.get("nc"),
|
||||
"names": tcfg.get("names"),
|
||||
"class_counts": class_by_task.get(task, {}),
|
||||
"packs": [],
|
||||
"drop_paths": {
|
||||
"inbox": str((root / "inbox" / task).resolve()),
|
||||
"sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "sources" / "<batch>").resolve()),
|
||||
},
|
||||
}
|
||||
for p in packs_reg.get("packs", []):
|
||||
pack_name = p["name"]
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
task_data = pack_dir / tcfg.get("task_dir", task)
|
||||
rep = report_splits.get((task, pack_name))
|
||||
if rep:
|
||||
def _pack_row(task_id: str, mode: str | None, mcfg: dict, pack_name: str, p: dict, task_data: Path) -> dict[str, Any]:
|
||||
rep_key = report_task_key(task_id, mode)
|
||||
rep = report_splits.get((rep_key, pack_name))
|
||||
split_counts = _count_split_images(task_data)
|
||||
scan_total = sum(split_counts.values())
|
||||
if rep and (scan_total == 0 or not _task_data_has_labels(task_data)):
|
||||
split_counts = {"train": rep["train"], "val": rep["val"], "test": rep["test"]}
|
||||
class_counts = report_classes.get(rep_key, class_by_task.get(rep_key, {}))
|
||||
bbox_points = _collect_bbox_points_sample(task_data) if _task_data_has_labels(task_data) else []
|
||||
label_distribution = {
|
||||
"class_counts": class_counts,
|
||||
"label_files": 0,
|
||||
"sampled": True,
|
||||
"total_boxes": sum(class_counts.values()) if class_counts else 0,
|
||||
"bbox_points": bbox_points,
|
||||
}
|
||||
else:
|
||||
label_distribution = _collect_pack_label_distribution(task_data, mcfg)
|
||||
if not label_distribution["class_counts"] and rep_key in report_classes:
|
||||
label_distribution["class_counts"] = report_classes[rep_key]
|
||||
if scan_total == 0 and rep:
|
||||
split_counts = {"train": rep["train"], "val": rep["val"], "test": rep["test"]}
|
||||
class_counts = report_classes.get(task, class_by_task.get(task, {}))
|
||||
bbox_points = _collect_bbox_points_sample(task_data)
|
||||
label_distribution = {
|
||||
"class_counts": class_counts,
|
||||
"label_files": 0,
|
||||
"sampled": True,
|
||||
"total_boxes": sum(class_counts.values()) if class_counts else 0,
|
||||
"bbox_points": bbox_points,
|
||||
}
|
||||
else:
|
||||
split_counts = _count_split_images(task_data)
|
||||
label_distribution = _collect_pack_label_distribution(task_data, tcfg)
|
||||
if not label_distribution["class_counts"] and task in report_classes:
|
||||
label_distribution["class_counts"] = report_classes[task]
|
||||
entry["packs"].append({
|
||||
"name": pack_name,
|
||||
"path": p.get("path"),
|
||||
"role": p.get("role"),
|
||||
"frozen": p.get("frozen", False),
|
||||
"enabled": pack_name in wf["projects"]["dms"].get("active_packs", []),
|
||||
"train_images": split_counts.get("train", 0),
|
||||
"val_images": split_counts.get("val", 0),
|
||||
"test_images": split_counts.get("test", 0),
|
||||
"class_counts": label_distribution["class_counts"],
|
||||
"label_files": label_distribution["label_files"],
|
||||
"total_boxes": label_distribution["total_boxes"],
|
||||
"sampled": label_distribution["sampled"],
|
||||
"bbox_points": label_distribution["bbox_points"],
|
||||
})
|
||||
if not entry["class_counts"] and task in report_classes:
|
||||
entry["class_counts"] = report_classes[task]
|
||||
return {
|
||||
"name": pack_name,
|
||||
"path": p.get("path"),
|
||||
"role": p.get("role"),
|
||||
"frozen": p.get("frozen", False),
|
||||
"enabled": pack_name in wf["projects"]["dms"].get("active_packs", []),
|
||||
"train_images": split_counts.get("train", 0),
|
||||
"val_images": split_counts.get("val", 0),
|
||||
"test_images": split_counts.get("test", 0),
|
||||
"class_counts": label_distribution["class_counts"],
|
||||
"label_files": label_distribution["label_files"],
|
||||
"total_boxes": label_distribution["total_boxes"],
|
||||
"sampled": label_distribution["sampled"],
|
||||
"bbox_points": label_distribution["bbox_points"],
|
||||
}
|
||||
|
||||
for task, entry in iter_catalog_tasks(reg):
|
||||
tcfg = reg["tasks"][task]
|
||||
if tcfg.get("type") == "multi":
|
||||
entry["drop_paths"] = {
|
||||
**{
|
||||
f"inbox_{mode}": str(inbox_dir(root, task, mode, reg).resolve())
|
||||
for mode in (tcfg.get("modes") or {})
|
||||
},
|
||||
"sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "<subdir>" / "sources" / "<batch>").resolve()),
|
||||
}
|
||||
for mode in (tcfg.get("modes") or {}):
|
||||
mcfg = get_mode_config(task, mode, reg)
|
||||
mode_entry = entry["modes"][mode]
|
||||
rep_key = report_task_key(task, mode)
|
||||
mode_entry["class_counts"] = class_by_task.get("isa_detect", class_by_task.get(rep_key, {})) if mode == "detect" else class_by_task.get("isa_class_0116", class_by_task.get(rep_key, {}))
|
||||
if not mode_entry["class_counts"] and rep_key in report_classes:
|
||||
mode_entry["class_counts"] = report_classes[rep_key]
|
||||
for p in packs_reg.get("packs", []):
|
||||
pack_name = p["name"]
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
task_data = resolve_task_data_for_scan(pack_dir, mcfg["task_dir"])
|
||||
mode_entry["packs"].append(_pack_row(task, mode, mcfg, pack_name, p, task_data))
|
||||
else:
|
||||
entry["drop_paths"] = {
|
||||
"inbox": str(inbox_dir(root, task, None, reg).resolve()),
|
||||
"sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "sources" / "<batch>").resolve()),
|
||||
}
|
||||
rep_key = task
|
||||
if not entry["class_counts"]:
|
||||
entry["class_counts"] = class_by_task.get(task, report_classes.get(task, {}))
|
||||
for p in packs_reg.get("packs", []):
|
||||
pack_name = p["name"]
|
||||
try:
|
||||
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
|
||||
except ValueError:
|
||||
continue
|
||||
task_data = resolve_task_data_for_scan(pack_dir, tcfg.get("task_dir", task))
|
||||
entry["packs"].append(_pack_row(task, None, tcfg, pack_name, p, task_data))
|
||||
if not entry["class_counts"] and task in report_classes:
|
||||
entry["class_counts"] = report_classes[task]
|
||||
out["dms"][task] = entry
|
||||
|
||||
out["dms"] = _normalize_catalog_dms(out["dms"], reg)
|
||||
|
||||
root = proj_root(wf, "lane")
|
||||
try:
|
||||
reg = load_pack_registry("lane", root, wf)
|
||||
@@ -718,6 +964,12 @@ def get_catalog(
|
||||
full_catalog, build_source = _build_catalog(wf, prefer_reports=not refresh)
|
||||
store_catalog_cache(sig, full_catalog, build_source=build_source)
|
||||
cache_meta = {"cached": False, "build_source": build_source}
|
||||
else:
|
||||
reg_path = proj_root(wf, "dms") / wf["projects"]["dms"]["registry"]
|
||||
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8")) if reg_path.is_file() else {}
|
||||
dms = _normalize_catalog_dms(full_catalog.get("dms") or {}, reg)
|
||||
if dms != full_catalog.get("dms"):
|
||||
full_catalog = {**full_catalog, "dms": dms}
|
||||
|
||||
result: dict[str, Any]
|
||||
if project == "dms" and task_or_pack:
|
||||
|
||||
91
platform/as_platform/data/ingest/dms_inbox_raw.py
Normal file
91
platform/as_platform/data/ingest/dms_inbox_raw.py
Normal file
@@ -0,0 +1,91 @@
|
||||
"""DMS inbox 原图批次(如 addw/ddaw:仅有 images/,待送标,无 YOLO/COCO 标注)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from as_platform.data.batch import count_images, dms_has_images
|
||||
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
|
||||
|
||||
|
||||
def _count_images(path: Path) -> int:
|
||||
return count_images(path)
|
||||
|
||||
|
||||
def _split_image_counts(root: Path) -> dict[str, int]:
|
||||
train = _count_images(root / "images" / "train")
|
||||
val = _count_images(root / "images" / "val")
|
||||
test = _count_images(root / "images" / "test")
|
||||
if train + val + test == 0:
|
||||
train = _count_images(root / "images")
|
||||
if train + val + test == 0 and root.name == "images":
|
||||
train = _count_images(root / "train")
|
||||
val = _count_images(root / "val")
|
||||
test = _count_images(root / "test")
|
||||
if train + val + test == 0:
|
||||
train = _count_images(root)
|
||||
if train + val + test == 0 and (root / "train").is_dir():
|
||||
train = _count_images(root / "train")
|
||||
val = _count_images(root / "val")
|
||||
test = _count_images(root / "test")
|
||||
return {"train": train, "val": val, "test": test}
|
||||
|
||||
|
||||
def _has_label_txts(root: Path) -> bool:
|
||||
for sub in ("labels", "labels/train"):
|
||||
d = root / sub
|
||||
if d.is_dir() and any(d.rglob("*.txt")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_inbox_raw_layout(root: Path) -> bool:
|
||||
if _has_label_txts(root):
|
||||
return False
|
||||
if dms_has_images(root):
|
||||
return True
|
||||
if root.name == "images" and (_count_images(root / "train") > 0 or _count_images(root) > 0):
|
||||
return True
|
||||
if (root / "train").is_dir() and _count_images(root / "train") > 0:
|
||||
return True
|
||||
if _count_images(root) > 0 and not (root / "images").is_dir() and not (root / "labels").is_dir():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class DmsInboxRawAdapter(IngestAdapter):
|
||||
format_id = "dms_inbox_raw"
|
||||
projects = ("dms",)
|
||||
|
||||
def can_handle(self, ctx: IngestContext) -> bool:
|
||||
return _is_inbox_raw_layout(ctx.source_path)
|
||||
|
||||
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
|
||||
root = ctx.source_path
|
||||
split_counts = _split_image_counts(root)
|
||||
sample_count = sum(split_counts.values())
|
||||
warnings: list[str] = []
|
||||
if sample_count == 0:
|
||||
warnings.append(
|
||||
"未找到图片;请填批次根目录(含 images/train/)或 images/、train/ 目录"
|
||||
)
|
||||
|
||||
artifacts: list[str] = []
|
||||
if (root / "images").is_dir():
|
||||
artifacts.append("images/")
|
||||
elif (root / "train").is_dir():
|
||||
artifacts.append("train/(将规范为 images/train)")
|
||||
elif root.name == "images":
|
||||
artifacts.append("images/")
|
||||
|
||||
return NormalizedDataset(
|
||||
format_id=self.format_id,
|
||||
project=ctx.project,
|
||||
task=ctx.task,
|
||||
source_path=str(root),
|
||||
split_counts=split_counts,
|
||||
sample_count=sample_count,
|
||||
annotation_count=0,
|
||||
artifacts=artifacts,
|
||||
warnings=warnings,
|
||||
extra={"stage_hint": "raw_pool"},
|
||||
)
|
||||
@@ -5,6 +5,7 @@ from pathlib import Path
|
||||
|
||||
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
|
||||
from as_platform.data.ingest.dms_coco import DmsCocoAdapter
|
||||
from as_platform.data.ingest.dms_inbox_raw import DmsInboxRawAdapter
|
||||
from as_platform.data.ingest.dms_yolo import DmsYoloAdapter
|
||||
from as_platform.data.ingest.lane_lines import LaneLinesAdapter
|
||||
from as_platform.data.ingest.lane_mask import LaneMaskAdapter
|
||||
@@ -17,6 +18,7 @@ class UnknownFormatError(ValueError):
|
||||
ADAPTERS: tuple[IngestAdapter, ...] = (
|
||||
DmsYoloAdapter(),
|
||||
DmsCocoAdapter(),
|
||||
DmsInboxRawAdapter(),
|
||||
LaneMaskAdapter(),
|
||||
LaneLinesAdapter(),
|
||||
)
|
||||
@@ -32,9 +34,15 @@ def detect_adapter(ctx: IngestContext) -> IngestAdapter:
|
||||
continue
|
||||
if adapter.can_handle(ctx):
|
||||
return adapter
|
||||
hint = ""
|
||||
if ctx.project == "dms":
|
||||
hint = (
|
||||
";DMS 送标/inbox 请使用批次根目录,且至少包含 images/train/*.jpg"
|
||||
"(或已标注的 images/+labels/、COCO annotations/)"
|
||||
)
|
||||
raise UnknownFormatError(
|
||||
f"unable to detect format for project={ctx.project}, task={ctx.task}, "
|
||||
f"source={ctx.source_path}. supported={available_formats(ctx.project)}"
|
||||
f"source={ctx.source_path}. supported={available_formats(ctx.project)}{hint}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -8,10 +8,14 @@ import uuid
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import BinaryIO
|
||||
from typing import Any, BinaryIO
|
||||
|
||||
import yaml
|
||||
|
||||
from as_platform.config import MANIFESTS
|
||||
from as_platform.data.batch import META_FILENAME, dms_has_images, enrich_batch, write_meta
|
||||
from as_platform.data.catalog_cache import invalidate_catalog_cache
|
||||
from as_platform.data.core import load_wf, proj_root
|
||||
from as_platform.data.ingest import inspect_uploaded_dataset
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import DatasetCandidate
|
||||
@@ -37,6 +41,7 @@ def create_uploaded_candidate(
|
||||
*,
|
||||
project: str,
|
||||
task: str | None,
|
||||
mode: str | None = None,
|
||||
original_name: str,
|
||||
upload_size_bytes: int,
|
||||
submitted_by_name: str | None,
|
||||
@@ -51,6 +56,7 @@ def create_uploaded_candidate(
|
||||
id=candidate_id,
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
status="uploaded",
|
||||
source_type="upload",
|
||||
original_name=original_name,
|
||||
@@ -82,15 +88,17 @@ def write_candidate_upload(candidate_id: str, stream: BinaryIO, chunk_size: int
|
||||
return str(path)
|
||||
|
||||
|
||||
def list_candidates(limit: int = 100) -> list[dict]:
|
||||
def list_candidates(*, offset: int = 0, limit: int = 20) -> dict[str, Any]:
|
||||
with session_scope() as db:
|
||||
rows = (
|
||||
db.query(DatasetCandidate)
|
||||
.order_by(DatasetCandidate.created_at.desc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
return [r.to_dict() for r in rows]
|
||||
q = db.query(DatasetCandidate).order_by(DatasetCandidate.created_at.desc())
|
||||
total = q.count()
|
||||
rows = q.offset(max(0, offset)).limit(max(1, limit)).all()
|
||||
return {
|
||||
"items": [r.to_dict() for r in rows],
|
||||
"total": total,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
|
||||
def get_candidate(candidate_id: str) -> dict | None:
|
||||
@@ -153,6 +161,8 @@ def analyze_uploaded_candidate(candidate_id: str) -> dict:
|
||||
|
||||
try:
|
||||
dataset_root = _extract_to_staging(upload_path, staging_dir)
|
||||
if project == "dms":
|
||||
dataset_root = _ensure_dms_inbox_layout(dataset_root)
|
||||
normalized = inspect_uploaded_dataset(project, task, dataset_root)
|
||||
report_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_file.write_text(json.dumps(normalized.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
@@ -177,3 +187,306 @@ def analyze_uploaded_candidate(candidate_id: str) -> dict:
|
||||
rec.error_message = str(e)
|
||||
db.flush()
|
||||
raise
|
||||
|
||||
|
||||
_SPLIT_DIR_NAMES = frozenset({"train", "val", "test"})
|
||||
_STRUCTURE_DIR_NAMES = frozenset({"images", "labels", "annotations"})
|
||||
|
||||
|
||||
def _dataset_root_from_dir(source_dir: Path) -> Path:
|
||||
"""解析批次根目录;勿把 images/ 或 train/ 误剥成更深层导致丢失 images 层。"""
|
||||
if not source_dir.is_dir():
|
||||
raise FileNotFoundError(f"not a directory: {source_dir}")
|
||||
subdirs = [p for p in source_dir.iterdir() if p.is_dir() and not p.name.startswith(".")]
|
||||
files = [p for p in source_dir.iterdir() if p.is_file()]
|
||||
if len(subdirs) == 1 and not files:
|
||||
only = subdirs[0]
|
||||
if only.name in _SPLIT_DIR_NAMES or only.name in _STRUCTURE_DIR_NAMES:
|
||||
return source_dir
|
||||
return only
|
||||
return source_dir
|
||||
|
||||
|
||||
def _ensure_dms_inbox_layout(root: Path) -> Path:
|
||||
"""将 dataset/train 布局规范为 …/images/train;已是批次根或 images/ 目录则不改动。"""
|
||||
if not root.is_dir():
|
||||
return root
|
||||
from as_platform.data.batch import count_images, dms_has_images
|
||||
|
||||
if dms_has_images(root):
|
||||
return root
|
||||
if root.name == "images" and count_images(root / "train") > 0:
|
||||
return root
|
||||
if count_images(root) > 0 and not any(
|
||||
(root / sub).is_dir() for sub in ("images", "train", "labels")
|
||||
):
|
||||
images_dir = root / "images"
|
||||
images_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest_train = images_dir / "train"
|
||||
dest_train.mkdir(parents=True, exist_ok=True)
|
||||
for item in list(root.iterdir()):
|
||||
if item.is_file() and item.suffix.lower() in {".jpg", ".jpeg", ".png", ".bmp", ".webp"}:
|
||||
shutil.move(str(item), str(dest_train / item.name))
|
||||
return root
|
||||
|
||||
train_dir = root / "train"
|
||||
if not train_dir.is_dir() or count_images(train_dir) == 0:
|
||||
return root
|
||||
images_dir = root / "images"
|
||||
if images_dir.is_dir() and count_images(images_dir / "train") > 0:
|
||||
return root
|
||||
images_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest_train = images_dir / "train"
|
||||
if dest_train.exists():
|
||||
shutil.rmtree(dest_train, ignore_errors=True)
|
||||
shutil.move(str(train_dir), str(dest_train))
|
||||
return root
|
||||
|
||||
|
||||
def analyze_directory_candidate(candidate_id: str, source_dir: Path | None = None) -> dict:
|
||||
"""分析目录型数据源(飞书 data_path / NAS),无需 zip 上传。"""
|
||||
upload_dir, staging_dir, report_file = _candidate_dirs(candidate_id)
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if not rec:
|
||||
raise ValueError(f"candidate not found: {candidate_id}")
|
||||
rec.status = "analyzing"
|
||||
rec.error_message = None
|
||||
db.flush()
|
||||
project = rec.project
|
||||
task = rec.task
|
||||
root = source_dir or Path(rec.upload_path)
|
||||
|
||||
if not root.is_dir():
|
||||
msg = f"source directory missing: {root}"
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if rec:
|
||||
rec.status = "failed"
|
||||
rec.error_message = msg
|
||||
raise FileNotFoundError(msg)
|
||||
|
||||
try:
|
||||
dataset_root = _dataset_root_from_dir(root)
|
||||
if staging_dir.exists():
|
||||
shutil.rmtree(staging_dir)
|
||||
shutil.copytree(dataset_root, staging_dir / "dataset", dirs_exist_ok=True)
|
||||
analyzed_root = staging_dir / "dataset"
|
||||
if project == "dms":
|
||||
analyzed_root = _ensure_dms_inbox_layout(analyzed_root)
|
||||
normalized = inspect_uploaded_dataset(project, task, analyzed_root)
|
||||
report_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_file.write_text(json.dumps(normalized.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if not rec:
|
||||
raise ValueError(f"candidate not found during finalize: {candidate_id}")
|
||||
rec.status = "analyzed"
|
||||
rec.analyzed_source_path = str(analyzed_root)
|
||||
rec.format_id = normalized.format_id
|
||||
rec.set_split_counts(normalized.split_counts)
|
||||
rec.set_quality(normalized.to_dict())
|
||||
rec.error_message = None
|
||||
db.flush()
|
||||
invalidate_catalog_cache()
|
||||
return normalized.to_dict()
|
||||
except Exception as e:
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if rec:
|
||||
rec.status = "failed"
|
||||
rec.error_message = str(e)
|
||||
db.flush()
|
||||
raise
|
||||
|
||||
|
||||
def create_directory_candidate(
|
||||
*,
|
||||
project: str,
|
||||
task: str | None,
|
||||
mode: str | None,
|
||||
source_dir: Path,
|
||||
source_type: str = "platform_delivery",
|
||||
external_id: str | None = None,
|
||||
feishu_record_id: str | None = None,
|
||||
) -> dict:
|
||||
"""为外部目录(平台送标申请 / 飞书台账)创建入湖候选。"""
|
||||
candidate_id = _new_candidate_id()
|
||||
upload_dir, _, _ = _candidate_dirs(candidate_id)
|
||||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||||
with session_scope() as db:
|
||||
rec = DatasetCandidate(
|
||||
id=candidate_id,
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
status="uploaded",
|
||||
source_type=source_type,
|
||||
original_name=source_dir.name,
|
||||
upload_path=str(source_dir.resolve()),
|
||||
upload_size_bytes=0,
|
||||
external_id=external_id,
|
||||
feishu_record_id=feishu_record_id,
|
||||
)
|
||||
db.add(rec)
|
||||
db.flush()
|
||||
return rec.to_dict()
|
||||
|
||||
|
||||
def create_feishu_directory_candidate(
|
||||
*,
|
||||
project: str,
|
||||
task: str | None,
|
||||
mode: str | None,
|
||||
source_dir: Path,
|
||||
external_id: str | None = None,
|
||||
feishu_record_id: str | None = None,
|
||||
) -> dict:
|
||||
"""为飞书台账行创建候选并指向 NAS/本地目录。"""
|
||||
return create_directory_candidate(
|
||||
project=project,
|
||||
task=task,
|
||||
mode=mode,
|
||||
source_dir=source_dir,
|
||||
source_type="feishu_bitable",
|
||||
external_id=external_id,
|
||||
feishu_record_id=feishu_record_id,
|
||||
)
|
||||
|
||||
|
||||
def _copy_tree_into(dest: Path, src: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for item in src.iterdir():
|
||||
target = dest / item.name
|
||||
if item.is_dir():
|
||||
if target.exists():
|
||||
_copy_tree_into(target, item)
|
||||
else:
|
||||
shutil.copytree(item, target)
|
||||
else:
|
||||
shutil.copy2(item, target)
|
||||
|
||||
|
||||
def _resolve_dms_inbox_dest(root: Path, reg: dict, task: str, mode: str | None, batch_name: str) -> Path:
|
||||
import sys
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
|
||||
scripts = WORKSPACE / "datasets" / "dms" / "scripts"
|
||||
if str(scripts) not in sys.path:
|
||||
sys.path.insert(0, str(scripts))
|
||||
from task_registry import inbox_dir, resolve_task_id
|
||||
|
||||
task_r, mode_r = resolve_task_id(task, mode)
|
||||
tcfg = (reg.get("tasks") or {}).get(task_r) or {}
|
||||
ib = inbox_dir(root, task_r, mode_r, reg)
|
||||
if tcfg.get("type") == "multi" and mode_r:
|
||||
if dms_has_images(ib) or (ib / META_FILENAME).is_file():
|
||||
return ib
|
||||
return ib / batch_name
|
||||
return ib / batch_name
|
||||
|
||||
|
||||
def promote_candidate_to_inbox(
|
||||
candidate_id: str,
|
||||
*,
|
||||
batch: str | None = None,
|
||||
mode: str | None = None,
|
||||
) -> dict:
|
||||
"""将 analyzed 候选数据复制到 inbox,并登记 batch.meta。"""
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if not rec:
|
||||
raise ValueError(f"candidate not found: {candidate_id}")
|
||||
if rec.status not in ("analyzed",):
|
||||
raise ValueError(f"candidate 状态须为 analyzed,当前: {rec.status}")
|
||||
if not rec.analyzed_source_path:
|
||||
raise ValueError("缺少 analyzed_source_path,请先完成分析")
|
||||
project = rec.project
|
||||
task = rec.task
|
||||
eff_mode = mode or rec.mode
|
||||
src = Path(rec.analyzed_source_path)
|
||||
cand_format_id = rec.format_id
|
||||
|
||||
if not src.is_dir():
|
||||
raise FileNotFoundError(f"分析目录不存在: {src}")
|
||||
|
||||
wf = load_wf()
|
||||
root = proj_root(wf, project)
|
||||
batch_name = batch or src.name or candidate_id.split("-", 1)[-1]
|
||||
|
||||
if project == "dms":
|
||||
if not task:
|
||||
raise ValueError("DMS 晋级需要 task")
|
||||
reg_path = root / wf["projects"]["dms"]["registry"]
|
||||
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
|
||||
tcfg = (reg.get("tasks") or {}).get(task) or {}
|
||||
if tcfg.get("type") == "multi" and not eff_mode:
|
||||
raise ValueError(f"任务 {task} 为 multi,须指定 mode(如 batch_0516)")
|
||||
dest = _resolve_dms_inbox_dest(root, reg, task, eff_mode, batch_name)
|
||||
reg_batch = eff_mode or batch_name
|
||||
else:
|
||||
dest = root / "inbox" / batch_name
|
||||
reg_batch = batch_name
|
||||
|
||||
if dest.exists() and any(dest.iterdir()):
|
||||
_copy_tree_into(dest, src)
|
||||
else:
|
||||
if dest.exists():
|
||||
shutil.rmtree(dest)
|
||||
shutil.copytree(src, dest)
|
||||
|
||||
row = enrich_batch(
|
||||
dest,
|
||||
project=project,
|
||||
task=task if project == "dms" else None,
|
||||
pack=None,
|
||||
batch=reg_batch,
|
||||
location="inbox",
|
||||
)
|
||||
fmt = row.get("format", "yolo")
|
||||
if cand_format_id == "dms_inbox_raw":
|
||||
fmt = "inbox_raw"
|
||||
meta_payload = {
|
||||
"schema": "huaxu-batch-v1",
|
||||
"project": project,
|
||||
"task": task,
|
||||
"batch": reg_batch,
|
||||
"stage": "raw_pool",
|
||||
"location": "inbox",
|
||||
"format": fmt,
|
||||
"counts": row.get("counts", {}),
|
||||
}
|
||||
if eff_mode:
|
||||
meta_payload["mode"] = eff_mode
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if rec:
|
||||
if rec.external_id:
|
||||
meta_payload["external_id"] = rec.external_id
|
||||
if rec.feishu_record_id:
|
||||
meta_payload["feishu_record_id"] = rec.feishu_record_id
|
||||
if rec.source_type and rec.source_type != "upload":
|
||||
meta_payload["source_type"] = rec.source_type
|
||||
write_meta(dest, meta_payload)
|
||||
meta = {**row, "stage": "raw_pool"}
|
||||
|
||||
with session_scope() as db:
|
||||
rec = db.get(DatasetCandidate, candidate_id)
|
||||
if rec:
|
||||
rec.status = "promoted"
|
||||
rec.inbox_path = str(dest)
|
||||
rec.promoted_batch = reg_batch
|
||||
if eff_mode:
|
||||
rec.mode = eff_mode
|
||||
db.flush()
|
||||
|
||||
invalidate_catalog_cache()
|
||||
return {
|
||||
"ok": True,
|
||||
"candidate_id": candidate_id,
|
||||
"inbox_path": str(dest),
|
||||
"batch": meta.get("batch", reg_batch),
|
||||
"stage": meta.get("stage"),
|
||||
"counts": meta.get("counts"),
|
||||
}
|
||||
|
||||
179
platform/as_platform/data/simulate.py
Normal file
179
platform/as_platform/data/simulate.py
Normal file
@@ -0,0 +1,179 @@
|
||||
"""世界模型仿真数据生成 — API 接口层。
|
||||
|
||||
实际生成逻辑由外部世界模型引擎完成(通过 subprocess / HTTP 调用)。
|
||||
本模块提供:任务提交、队列管理、状态追踪、结果入库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
from as_platform.db.engine import session_scope
|
||||
|
||||
SIM_JOBS_DIR = WORKSPACE / "manifests" / "simulation_jobs"
|
||||
SIM_OUTPUT_ROOT = WORKSPACE / "datasets" / "dms" / "simulated"
|
||||
|
||||
SCENE_TEMPLATES = {
|
||||
"urban_highway": {"label": "城市快速路", "desc": "多车道高速公路场景"},
|
||||
"urban_street": {"label": "城市街道", "desc": "有交通灯和路口的城市道路"},
|
||||
"rural_road": {"label": "乡村道路", "desc": "双车道乡村公路"},
|
||||
"tunnel": {"label": "隧道", "desc": "隧道内光照变化场景"},
|
||||
"night_city": {"label": "夜间城市", "desc": "低光照城市道路"},
|
||||
"rain_highway": {"label": "雨天高速", "desc": "雨天湿滑路面"},
|
||||
"fog_rural": {"label": "雾天乡村", "desc": "大雾低能见度"},
|
||||
}
|
||||
|
||||
CAMERA_PRESETS = {
|
||||
"truck_front": {"label": "卡车前视", "height": 2.5, "fov": 75, "pitch": -5},
|
||||
"truck_side": {"label": "卡车侧视", "height": 2.5, "fov": 100, "pitch": 0},
|
||||
"car_front": {"label": "轿车前视", "height": 1.2, "fov": 60, "pitch": -3},
|
||||
"car_wide": {"label": "轿车广角", "height": 1.2, "fov": 120, "pitch": 0},
|
||||
}
|
||||
|
||||
OBJECT_CLASSES = ["Pedestrain", "Car", "Truck", "Bus", "Motor-vehicles", "Tricycle", "cones"]
|
||||
|
||||
|
||||
def list_jobs(offset: int = 0, limit: int = 20) -> dict[str, Any]:
|
||||
SIM_JOBS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
jobs = []
|
||||
for f in sorted(SIM_JOBS_DIR.glob("*.json"), reverse=True):
|
||||
try:
|
||||
data = json.loads(f.read_text())
|
||||
data["_id"] = f.stem
|
||||
if data.get("status") != "archived":
|
||||
jobs.append(data)
|
||||
except Exception:
|
||||
pass
|
||||
total = len(jobs)
|
||||
return {"items": jobs[offset:offset + limit], "total": total}
|
||||
|
||||
|
||||
def submit_job(params: dict[str, Any], user_name: str = "") -> dict[str, Any]:
|
||||
job_id = f"sim-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
scene = params.get("scene", "urban_highway")
|
||||
camera = params.get("camera", "truck_front")
|
||||
weather = params.get("weather", "clear")
|
||||
objects = params.get("objects", OBJECT_CLASSES[:4])
|
||||
density = params.get("density", "medium")
|
||||
count = min(params.get("count", 100), 5000)
|
||||
note = params.get("note", "")
|
||||
fov_variant = params.get("fov_variant", False) # 是否生成多FOV变体
|
||||
|
||||
scene_info = SCENE_TEMPLATES.get(scene, {"label": scene})
|
||||
cam_info = CAMERA_PRESETS.get(camera, {"label": camera})
|
||||
|
||||
job = {
|
||||
"id": job_id,
|
||||
"status": "queued",
|
||||
"created_at": now,
|
||||
"submitted_by": user_name,
|
||||
"params": {
|
||||
"scene": scene,
|
||||
"scene_label": scene_info["label"],
|
||||
"camera": camera,
|
||||
"camera_label": cam_info["label"],
|
||||
"camera_height": cam_info.get("height", 2.5),
|
||||
"camera_fov": cam_info.get("fov", 75),
|
||||
"weather": weather,
|
||||
"objects": objects,
|
||||
"density": density,
|
||||
"count": count,
|
||||
"fov_variant": fov_variant,
|
||||
"note": note,
|
||||
},
|
||||
"result": None,
|
||||
"batch_registered": False,
|
||||
}
|
||||
|
||||
SIM_JOBS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(SIM_JOBS_DIR / f"{job_id}.json").write_text(json.dumps(job, ensure_ascii=False, indent=2))
|
||||
|
||||
# Queue actual generation (mock for now — replace with real world model call)
|
||||
_trigger_generation(job_id)
|
||||
|
||||
return job
|
||||
|
||||
|
||||
def get_job(job_id: str) -> dict[str, Any] | None:
|
||||
f = SIM_JOBS_DIR / f"{job_id}.json"
|
||||
if not f.is_file():
|
||||
return None
|
||||
data = json.loads(f.read_text())
|
||||
data["_id"] = f.stem
|
||||
return data
|
||||
|
||||
|
||||
def get_job_images(job_id: str, offset: int = 0, limit: int = 60) -> dict[str, Any]:
|
||||
out_dir = SIM_OUTPUT_ROOT / job_id / "images"
|
||||
if not out_dir.is_dir():
|
||||
return {"items": [], "total": 0}
|
||||
imgs = sorted(out_dir.glob("*.jpg")) + sorted(out_dir.glob("*.png"))
|
||||
total = len(imgs)
|
||||
page = imgs[offset:offset + limit]
|
||||
return {
|
||||
"items": [{"name": p.name, "path": str(p.relative_to(SIM_OUTPUT_ROOT))} for p in page],
|
||||
"total": total,
|
||||
}
|
||||
|
||||
|
||||
def ingest_job_to_batch(job_id: str, task: str = "adas", user_name: str = "") -> dict[str, Any]:
|
||||
"""将仿真生成的数据注册为批次,直接入库。"""
|
||||
job = get_job(job_id)
|
||||
if not job:
|
||||
return {"ok": False, "error": "Job not found"}
|
||||
|
||||
out_dir = SIM_OUTPUT_ROOT / job_id
|
||||
if not out_dir.is_dir():
|
||||
return {"ok": False, "error": "生成数据不存在"}
|
||||
|
||||
# Register as batch
|
||||
from as_platform.data.core import register_batch
|
||||
batch_name = f"sim_{job_id}"
|
||||
try:
|
||||
register_batch(None, "dms", task, batch_name, stage="returned", location="inbox")
|
||||
# Update job status
|
||||
job["status"] = "ingested"
|
||||
job["batch_registered"] = True
|
||||
job["batch_name"] = batch_name
|
||||
(SIM_JOBS_DIR / f"{job_id}.json").write_text(json.dumps(job, ensure_ascii=False, indent=2))
|
||||
return {"ok": True, "batch": batch_name, "task": task}
|
||||
except Exception as e:
|
||||
return {"ok": False, "error": str(e)}
|
||||
|
||||
|
||||
def _trigger_generation(job_id: str) -> None:
|
||||
"""触发实际生成(当前为 mock)。替换为真实世界模型调用。"""
|
||||
import threading
|
||||
|
||||
def _run():
|
||||
try:
|
||||
# TODO: 替换为真实世界模型调用
|
||||
# 例如: subprocess.run(["python", "world_model/generate.py", "--job", job_id])
|
||||
_update_status(job_id, "running")
|
||||
# Mock: create output directory but no actual images
|
||||
out_dir = SIM_OUTPUT_ROOT / job_id / "images"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
# Mock 完成
|
||||
_update_status(job_id, "completed")
|
||||
except Exception as e:
|
||||
_update_status(job_id, "failed", str(e))
|
||||
|
||||
threading.Thread(target=_run, daemon=True, name=f"sim-{job_id}").start()
|
||||
|
||||
|
||||
def _update_status(job_id: str, status: str, error: str = "") -> None:
|
||||
f = SIM_JOBS_DIR / f"{job_id}.json"
|
||||
if f.is_file():
|
||||
data = json.loads(f.read_text())
|
||||
data["status"] = status
|
||||
if error:
|
||||
data["error"] = error
|
||||
if status == "completed":
|
||||
data["completed_at"] = datetime.now(timezone.utc).isoformat()
|
||||
f.write_text(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
161
platform/as_platform/data/versions.py
Normal file
161
platform/as_platform/data/versions.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""数据集版本管理 — snapshot / diff / lineage。版本以 JSON 文件存储在 datasets/<project>/versions/。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import WORKSPACE
|
||||
from as_platform.data.core import get_catalog
|
||||
|
||||
|
||||
def _versions_dir(project: str) -> Path:
|
||||
return WORKSPACE / "datasets" / project / "versions"
|
||||
|
||||
|
||||
def list_versions(project: str = "dms") -> list[dict[str, Any]]:
|
||||
"""列出所有版本,按时间倒序。"""
|
||||
vdir = _versions_dir(project)
|
||||
if not vdir.is_dir():
|
||||
return []
|
||||
versions: list[dict[str, Any]] = []
|
||||
for f in sorted(vdir.glob("*.json"), reverse=True):
|
||||
try:
|
||||
data = json.loads(f.read_text(encoding="utf-8"))
|
||||
data["_id"] = f.stem
|
||||
versions.append(data)
|
||||
except Exception:
|
||||
pass
|
||||
return versions
|
||||
|
||||
|
||||
def get_version(project: str, version_id: str) -> dict[str, Any] | None:
|
||||
vf = _versions_dir(project) / f"{version_id}.json"
|
||||
if not vf.is_file():
|
||||
return None
|
||||
data = json.loads(vf.read_text(encoding="utf-8"))
|
||||
data["_id"] = vf.stem
|
||||
return data
|
||||
|
||||
|
||||
def create_snapshot(project: str, description: str = "", author: str = "") -> dict[str, Any]:
|
||||
"""基于当前 catalog 创建数据集快照。"""
|
||||
catalog = get_catalog(refresh=True)
|
||||
dms = catalog.get("dms", {})
|
||||
lane = catalog.get("lane", {})
|
||||
|
||||
# Collect pack summary from catalog
|
||||
packs_summary: dict[str, dict[str, Any]] = {}
|
||||
all_batches: list[str] = []
|
||||
total_images = 0
|
||||
total_labels = 0
|
||||
|
||||
if project == "dms":
|
||||
for task_id, entry in dms.items():
|
||||
entry_d = entry if isinstance(entry, dict) else {}
|
||||
for p in entry_d.get("packs", []) or []:
|
||||
pname = p.get("name", "")
|
||||
if pname not in packs_summary:
|
||||
packs_summary[pname] = {
|
||||
"train_images": 0, "val_images": 0, "test_images": 0,
|
||||
"class_counts": {}, "tasks": [],
|
||||
}
|
||||
s = packs_summary[pname]
|
||||
s["train_images"] += p.get("train_images", 0) or 0
|
||||
s["val_images"] += p.get("val_images", 0) or 0
|
||||
s["test_images"] += p.get("test_images", 0) or 0
|
||||
if task_id not in s["tasks"]:
|
||||
s["tasks"].append(task_id)
|
||||
total_images += (p.get("train_images", 0) or 0) + (p.get("val_images", 0) or 0)
|
||||
total_labels += p.get("label_files", 0) or 0
|
||||
|
||||
# Collect batch names from pending report
|
||||
from as_platform.data.core import get_pending_report
|
||||
report = get_pending_report()
|
||||
for b in report.get("batches", []) or []:
|
||||
if isinstance(b, dict) and b.get("project") == "dms":
|
||||
all_batches.append(b.get("batch", ""))
|
||||
|
||||
# Determine version number
|
||||
existing = list_versions(project)
|
||||
version_num = len(existing) + 1
|
||||
version_id = f"v{version_num}"
|
||||
|
||||
# Get parent version
|
||||
parent_id = existing[0]["_id"] if existing else None
|
||||
|
||||
# Compute diff if parent exists
|
||||
diff: dict[str, Any] | None = None
|
||||
if parent_id and existing:
|
||||
parent = existing[0]
|
||||
parent_packs = set((parent.get("packs") or {}).keys())
|
||||
current_packs = set(packs_summary.keys())
|
||||
parent_batches = set(parent.get("batches", []) or [])
|
||||
current_batches = set(all_batches)
|
||||
diff = {
|
||||
"added_packs": sorted(current_packs - parent_packs),
|
||||
"removed_packs": sorted(parent_packs - current_packs),
|
||||
"added_batches": sorted(current_batches - parent_batches),
|
||||
"removed_batches": sorted(parent_batches - current_batches),
|
||||
}
|
||||
|
||||
snapshot = {
|
||||
"version_id": version_id,
|
||||
"project": project,
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"description": description,
|
||||
"author": author,
|
||||
"parent_version": parent_id,
|
||||
"summary": {
|
||||
"packs_count": len(packs_summary),
|
||||
"total_images": total_images,
|
||||
"total_labels": total_labels,
|
||||
"batches_count": len(all_batches),
|
||||
},
|
||||
"packs": packs_summary,
|
||||
"batches": sorted(all_batches),
|
||||
"diff": diff,
|
||||
}
|
||||
|
||||
# Save to file
|
||||
vdir = _versions_dir(project)
|
||||
vdir.mkdir(parents=True, exist_ok=True)
|
||||
(vdir / f"{version_id}.json").write_text(
|
||||
json.dumps(snapshot, ensure_ascii=False, indent=2, default=str),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
snapshot["_id"] = version_id
|
||||
return snapshot
|
||||
|
||||
|
||||
def diff_versions(project: str, v1: str, v2: str) -> dict[str, Any]:
|
||||
"""对比两个版本。"""
|
||||
a = get_version(project, v1)
|
||||
b = get_version(project, v2)
|
||||
if not a or not b:
|
||||
return {"error": "版本不存在"}
|
||||
|
||||
a_packs = set((a.get("packs") or {}).keys())
|
||||
b_packs = set((b.get("packs") or {}).keys())
|
||||
|
||||
pack_changes: list[dict[str, Any]] = []
|
||||
for pname in a_packs | b_packs:
|
||||
pa = (a.get("packs") or {}).get(pname, {})
|
||||
pb = (b.get("packs") or {}).get(pname, {})
|
||||
if pa != pb:
|
||||
pack_changes.append({
|
||||
"pack": pname,
|
||||
"v1": {"train": pa.get("train_images", 0), "val": pa.get("val_images", 0)},
|
||||
"v2": {"train": pb.get("train_images", 0), "val": pb.get("val_images", 0)},
|
||||
})
|
||||
|
||||
return {
|
||||
"v1": {"id": v1, "created": a.get("created_at"), "total": a.get("summary", {}).get("total_images", 0)},
|
||||
"v2": {"id": v2, "created": b.get("created_at"), "total": b.get("summary", {}).get("total_images", 0)},
|
||||
"added_packs": sorted(b_packs - a_packs),
|
||||
"removed_packs": sorted(a_packs - b_packs),
|
||||
"pack_changes": pack_changes,
|
||||
"image_delta": (b.get("summary", {}).get("total_images", 0) or 0) - (a.get("summary", {}).get("total_images", 0) or 0),
|
||||
}
|
||||
Reference in New Issue
Block a user