feat: HSAP platform v2 — modular navigation, quality review, audit log, world model simulation

Major changes:
- New frontend (platform/web/): Vite + React 18 + TypeScript + Tailwind
- 4-module navigation: 数据送标 / 模型管理 / 车队管理 / 系统管理
- Data catalog with charts (DMS/ADAS/Lane 3-tab view)
- Quality review workflow (标注质检): Good/Fine/Bad scoring with auto-advance
- Audit enhancements: batch operations, rejection categories, Feishu notifications
- Operation audit log (操作日志)
- World model simulation studio (仿真工坊)
- Dataset version management with snapshots and diff
- ADAS 7-class dataset integration (138K images organized + compressed)
- User management with Feishu integration and pagination
- CRUD/search/filter on all pages, card layout redesign
- PIL-optimized image overlay rendering
- Auto-snapshot on build, in_review workflow stage
- Removed embedded algorithm code (now in workspace)
This commit is contained in:
2026-06-03 11:40:21 +08:00
parent 7c43b44c57
commit e72bc061c5
5487 changed files with 979207 additions and 6197 deletions

View File

@@ -15,6 +15,7 @@ from as_platform.data.lake import (
get_candidate,
link_candidate_analysis_job,
list_candidates,
promote_candidate_to_inbox,
write_candidate_upload,
)
@@ -34,4 +35,5 @@ __all__ = [
"get_candidate",
"link_candidate_analysis_job",
"analyze_uploaded_candidate",
"promote_candidate_to_inbox",
]

View File

@@ -13,7 +13,7 @@ from as_platform.config import WORKSPACE
CATALOG_CACHE_FILE = WORKSPACE / "manifests" / "catalog_cache.json"
CATALOG_CACHE_TTL_SEC = int(os.environ.get("AS_CATALOG_CACHE_TTL_SEC", "300"))
CATALOG_USE_REPORTS = os.environ.get("AS_CATALOG_USE_REPORTS", "1").lower() in ("1", "true", "yes")
CATALOG_CACHE_VERSION = 3
CATALOG_CACHE_VERSION = 7
REPORTS_DIR = WORKSPACE / "reports"
DMS_SUMMARY_CSV = REPORTS_DIR / "dms_task_image_summary.csv"
@@ -111,8 +111,14 @@ def build_catalog_signature(wf: dict, proj_root_fn) -> dict[str, Any]:
import yaml
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
for task in (reg.get("tasks") or {}).keys():
dirs.append(_dir_fingerprint(root / "inbox" / task))
for task, tcfg in (reg.get("tasks") or {}).items():
if tcfg.get("type") == "multi":
for mcfg in (tcfg.get("modes") or {}).values():
rel = mcfg.get("inbox", "")
if rel:
dirs.append(_dir_fingerprint(root / rel))
else:
dirs.append(_dir_fingerprint(root / (tcfg.get("inbox") or f"inbox/{task}")))
if pname == "lane":
dirs.append(_dir_fingerprint(_pack_registry_path("lane", root, wf), scan_children=False))
@@ -140,13 +146,41 @@ def save_disk_cache(payload: dict[str, Any]) -> None:
CATALOG_CACHE_FILE.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
def _iter_dms_packs(task_entry: dict[str, Any]):
modes = task_entry.get("modes")
if isinstance(modes, dict):
for mode_entry in modes.values():
for pack in mode_entry.get("packs") or []:
if isinstance(pack, dict):
yield pack, mode_entry.get("type")
return
for pack in task_entry.get("packs") or []:
if isinstance(pack, dict):
yield pack, task_entry.get("type")
def _entry_needs_bbox_refresh(entry: dict[str, Any]) -> bool:
if entry.get("type") == "classify":
return False
has_class = bool(entry.get("class_counts"))
any_pts = False
any_boxes = False
for pack in entry.get("packs") or []:
if pack.get("bbox_points"):
any_pts = True
if int(pack.get("total_boxes") or 0) > 0:
any_boxes = True
return not any_pts and (any_boxes or has_class)
def _catalog_has_empty_bbox(catalog: dict[str, Any]) -> bool:
for task in (catalog.get("dms") or {}).values():
for pack in task.get("packs") or []:
boxes = int(pack.get("total_boxes") or 0)
pts = pack.get("bbox_points") or []
if boxes > 0 and not pts:
return True
if task.get("type") == "multi":
for mode_entry in (task.get("modes") or {}).values():
if _entry_needs_bbox_refresh(mode_entry):
return True
elif _entry_needs_bbox_refresh(task):
return True
return False
@@ -198,7 +232,8 @@ def store_catalog_cache(signature: dict[str, Any], data: dict[str, Any], *, buil
"data": data,
}
_CATALOG_MEM_CACHE = payload
save_disk_cache(payload)
if not _catalog_has_empty_bbox(data):
save_disk_cache(payload)
return payload

View File

@@ -10,8 +10,8 @@ from typing import Any
import yaml
from as_platform.config import WORKSPACE, LANE_DATA_VIZ_ENABLED
from as_platform.data.batch import META_FILENAME, enrich_batch, write_meta
from as_platform.config import WORKSPACE, WORKSPACE_ROOT, LANE_DATA_VIZ_ENABLED
from as_platform.data.batch import META_FILENAME, dms_has_images, enrich_batch, write_meta
from as_platform.data.catalog_cache import (
build_catalog_signature,
get_cached_catalog,
@@ -20,6 +20,30 @@ from as_platform.data.catalog_cache import (
store_catalog_cache,
)
import sys as _sys
def _dms_scripts_dir() -> Path:
for cand in (
WORKSPACE / "datasets" / "dms" / "scripts",
WORKSPACE / "datasets" / "dms.embedded.bak" / "scripts",
):
if (cand / "task_registry.py").is_file():
return cand
return WORKSPACE / "datasets" / "dms" / "scripts"
_DMS_SCRIPTS = _dms_scripts_dir()
if str(_DMS_SCRIPTS) not in _sys.path:
_sys.path.insert(0, str(_DMS_SCRIPTS))
from task_registry import ( # noqa: E402
DOMAIN_LABELS,
get_mode_config,
inbox_dir,
iter_catalog_tasks,
report_task_key,
task_defs_for_pending,
)
MAX_LABEL_FILES_PER_PACK = 2000
MAX_BBOX_POINTS_PER_PACK = 1500
MAX_LANE_MASK_SAMPLES_PER_PACK = 500
@@ -60,8 +84,36 @@ def resolve_pack(project: str, root: Path, wf: dict, name: str) -> str:
raise ValueError(f"[{project}] 未知包: {name},已登记: {known}")
def _pack_dir_usable(path: Path) -> bool:
try:
return path.is_dir() and any(path.iterdir())
except OSError:
return False
def _dms_workspace_pack_dir(pack_name: str) -> Path | None:
ws = WORKSPACE_ROOT
if ws is None:
return None
cand = ws / "DMS/DATASET/packs" / pack_name
return cand if _pack_dir_usable(cand) else None
def resolve_pack_dir(project: str, root: Path, wf: dict, name: str) -> Path:
return (root / resolve_pack(project, root, wf, name)).resolve()
rel = resolve_pack(project, root, wf, name)
primary = root / rel
if _pack_dir_usable(primary):
return primary.resolve()
reg = load_pack_registry(project, root, wf)
pack_name = reg.get("aliases", {}).get(name, name)
if project == "dms":
fallback = _dms_workspace_pack_dir(pack_name)
if fallback is not None:
return fallback
try:
return primary.resolve()
except OSError:
return primary
def _read_jsonl_tail(path: Path, n: int = 10) -> list[dict]:
@@ -111,28 +163,66 @@ def get_pending_report(wf: dict | None = None) -> dict[str, Any]:
ingest_log = root / "manifests" / "ingest_log.jsonl"
proj["recent_ingest"] = _read_jsonl_tail(ingest_log, 10)
for task, tcfg in reg.get("tasks", {}).items():
proj["task_defs"][task] = {
"type": tcfg.get("type"),
"nc": tcfg.get("nc"),
"names": tcfg.get("names"),
"task_dir": tcfg.get("task_dir", task),
}
inbox_batches: list[str] = []
ib = root / "inbox" / task
if ib.is_dir():
inbox_batches = [
d.name for d in ib.iterdir()
if d.is_dir() and not d.name.startswith(".")
]
proj["task_defs"] = task_defs_for_pending(reg)
sources_pending: dict[str, list[str]] = {}
def _scan_task_inbox(task_id: str, mode: str | None) -> list[str]:
ib = inbox_dir(root, task_id, mode, reg)
if not ib.is_dir():
return []
subdirs = [
d.name for d in ib.iterdir() if d.is_dir() and not d.name.startswith(".")
]
if subdirs:
return subdirs
tcfg = reg.get("tasks", {}).get(task_id, {})
if tcfg.get("type") == "multi" and mode:
if dms_has_images(ib):
return [mode]
return []
def _multi_inbox_batch_rows(task: str, mode: str) -> list[dict[str, Any]]:
"""multi 任务inbox 可能即批次根目录(如 inbox/dam/batch_0516"""
ib = inbox_dir(root, task, mode, reg)
if not ib.is_dir():
return []
rows: list[dict[str, Any]] = []
subdirs = [
d.name for d in ib.iterdir() if d.is_dir() and not d.name.startswith(".")
]
if subdirs:
for batch_name in subdirs:
row = enrich_batch(
ib / batch_name,
project="dms",
task=task,
pack=None,
batch=batch_name,
location="inbox",
)
row["mode"] = mode
rows.append(row)
elif dms_has_images(ib) or (ib / META_FILENAME).is_file():
row = enrich_batch(
ib,
project="dms",
task=task,
pack=None,
batch=mode,
location="inbox",
)
row["mode"] = mode
rows.append(row)
return rows
def _scan_sources(task_id: str, mode: str | None) -> dict[str, list[str]]:
mcfg = get_mode_config(task_id, mode, reg)
pending: dict[str, list[str]] = {}
for pack_name in all_names:
try:
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
except ValueError:
continue
src_root = pack_dir / tcfg["task_dir"] / src_sub
src_root = pack_dir / mcfg["task_dir"] / src_sub
if src_root.is_dir():
batches = [
d.name for d in src_root.iterdir()
@@ -141,44 +231,63 @@ def get_pending_report(wf: dict | None = None) -> dict[str, Any]:
and not d.name.startswith(".")
]
if batches:
sources_pending[pack_name] = batches
pending[pack_name] = batches
return pending
proj["tasks"][task] = {
"inbox": inbox_batches,
"sources": sources_pending,
}
for batch_name in inbox_batches:
batch_dir = ib / batch_name
report["batches"].append(
enrich_batch(
batch_dir,
project="dms",
task=task,
pack=None,
batch=batch_name,
location="inbox",
)
)
for pack_name, batch_list in sources_pending.items():
try:
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
except ValueError:
continue
src_root = pack_dir / tcfg["task_dir"] / src_sub
for batch_name in batch_list:
batch_dir = src_root / batch_name
for task, tcfg in reg.get("tasks", {}).items():
if tcfg.get("type") == "multi":
task_inbox: dict[str, list[str]] = {}
task_sources: dict[str, dict[str, list[str]]] = {}
for mode in (tcfg.get("modes") or {}):
task_inbox[mode] = _scan_task_inbox(task, mode)
task_sources[mode] = _scan_sources(task, mode)
for row in _multi_inbox_batch_rows(task, mode):
report["batches"].append(row)
for pack_name, batch_list in task_sources[mode].items():
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
src_root = pack_dir / get_mode_config(task, mode, reg)["task_dir"] / src_sub
for batch_name in batch_list:
row = enrich_batch(
src_root / batch_name,
project="dms",
task=task,
pack=pack_name,
batch=batch_name,
location="sources",
)
row["mode"] = mode
report["batches"].append(row)
proj["tasks"][task] = {"inbox": task_inbox, "sources": task_sources}
else:
inbox_batches = _scan_task_inbox(task, None)
sources_pending = _scan_sources(task, None)
proj["tasks"][task] = {"inbox": inbox_batches, "sources": sources_pending}
ib = inbox_dir(root, task, None, reg)
for batch_name in inbox_batches:
report["batches"].append(
enrich_batch(
batch_dir,
ib / batch_name,
project="dms",
task=task,
pack=pack_name,
pack=None,
batch=batch_name,
location="sources",
location="inbox",
)
)
for pack_name, batch_list in sources_pending.items():
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
src_root = pack_dir / tcfg["task_dir"] / src_sub
for batch_name in batch_list:
report["batches"].append(
enrich_batch(
src_root / batch_name,
project="dms",
task=task,
pack=pack_name,
batch=batch_name,
location="sources",
)
)
if pname == "lane":
proj["packs"] = {}
@@ -297,7 +406,7 @@ def _count_images_in_dir(img_dir: Path) -> int:
try:
with os.scandir(img_dir) as it:
for entry in it:
if not entry.is_file(follow_symlinks=False):
if not entry.is_file(follow_symlinks=True):
continue
if Path(entry.name).suffix.lower() in IMAGE_EXTS:
total += 1
@@ -325,8 +434,8 @@ def _iter_label_files(label_dirs: list[Path]):
continue
try:
with os.scandir(label_dir) as it:
stack = [entry.path for entry in it if entry.is_dir(follow_symlinks=False)]
files = [entry.path for entry in it if entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt")]
stack = [entry.path for entry in it if entry.is_dir(follow_symlinks=True)]
files = [entry.path for entry in it if entry.is_file(follow_symlinks=True) and entry.name.endswith(".txt")]
except OSError:
continue
for fp in files:
@@ -336,9 +445,9 @@ def _iter_label_files(label_dirs: list[Path]):
try:
with os.scandir(current) as it:
for entry in it:
if entry.is_dir(follow_symlinks=False):
if entry.is_dir(follow_symlinks=True):
stack.append(entry.path)
elif entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt"):
elif entry.is_file(follow_symlinks=True) and entry.name.endswith(".txt"):
yield Path(entry.path)
except OSError:
continue
@@ -348,6 +457,49 @@ def _label_dirs_for_task(task_data: Path) -> list[Path]:
return [task_data / "labels" / "train", task_data / "labels" / "val", task_data / "labels"]
def _task_data_has_labels(task_data: Path) -> bool:
return any(p.is_dir() for p in _label_dirs_for_task(task_data))
def resolve_task_data_for_scan(pack_dir: Path, task_dir: str) -> Path:
"""解析可扫描的 task 数据目录。
DAM 等批次在宿主机上常为指向绝对路径的 symlinkDocker 内 resolve() 会落到
容器外路径导致 is_dir() 为 False、bbox 采样为空。此处回映射到 AS_WORKSPACE_ROOT
并尝试 pack 级 _dam_stash_* 目录。
"""
task_data = pack_dir / task_dir
if _task_data_has_labels(task_data):
return task_data
if task_data.is_symlink():
try:
raw = os.readlink(task_data)
link = Path(raw)
candidates: list[Path] = []
if link.is_absolute():
s = str(link).replace("\\", "/")
for marker in ("DMS/DATASET/", "DMS/DATASET"):
idx = s.find(marker)
if idx >= 0 and WORKSPACE_ROOT:
candidates.append(WORKSPACE_ROOT / s[idx:])
else:
candidates.append((task_data.parent / link).resolve())
for cand in candidates:
if _task_data_has_labels(cand):
return cand
except OSError:
pass
leaf = Path(task_dir).name
if leaf.startswith("batch_"):
stash = pack_dir / f"_dam_stash_{leaf.split('_', 1)[1]}"
if _task_data_has_labels(stash):
return stash
return task_data
def _parse_bbox_wh(parts: list[str]) -> list[float] | None:
if len(parts) < 5:
return None
@@ -574,16 +726,80 @@ def _catalog_signature(wf: dict) -> dict[str, Any]:
return build_catalog_signature(wf, proj_root)
def _merge_multi_task_entry(
dms: dict[str, Any],
task_id: str,
tcfg: dict,
legacy_modes: dict[str, dict],
) -> None:
entry = dms.setdefault(task_id, {})
entry["type"] = "multi"
entry["domain"] = tcfg.get("domain", "dms")
entry["label"] = tcfg.get("label", task_id)
entry["domain_label"] = DOMAIN_LABELS.get(tcfg.get("domain", "dms"), tcfg.get("domain", "dms"))
modes = entry.setdefault("modes", {})
for mode, mcfg in (tcfg.get("modes") or {}).items():
mode_entry = modes.setdefault(mode, {})
mode_entry.setdefault("label", mcfg.get("label", mode))
mode_entry.setdefault("type", mcfg.get("type", "detect"))
mode_entry.setdefault("nc", mcfg.get("nc"))
mode_entry.setdefault("names", mcfg.get("names"))
mode_entry.setdefault("packs", [])
mode_entry.setdefault("class_counts", {})
if mode in legacy_modes:
leg = legacy_modes[mode]
if not mode_entry.get("packs") and leg.get("packs"):
mode_entry["packs"] = leg["packs"]
if not mode_entry.get("class_counts") and leg.get("class_counts"):
mode_entry["class_counts"] = leg["class_counts"]
stray = entry.pop("packs", None)
first_mode = next(iter(modes), None)
if stray and isinstance(stray, list) and first_mode and not modes[first_mode].get("packs"):
modes[first_mode]["packs"] = stray
entry.pop("nc", None)
entry.pop("names", None)
def _normalize_catalog_dms(dms: dict[str, Any], reg: dict) -> dict[str, Any]:
"""修复旧缓存isa/isa_class→forwarddam_0417→dam补全 multi 结构。"""
if not dms:
return dms
tasks_cfg = (reg or {}).get("tasks") or {}
legacy_forward = {}
for old_key, mode in (("isa", "detect"), ("isa_class", "classify")):
if old_key in dms:
legacy_forward[mode] = dms.pop(old_key)
fwd_cfg = tasks_cfg.get("forward") or {}
if fwd_cfg.get("type") == "multi" or legacy_forward:
_merge_multi_task_entry(dms, "forward", fwd_cfg, legacy_forward)
legacy_dam: dict[str, dict] = {}
if "dam_0417" in dms:
legacy_dam["batch_0417"] = dms.pop("dam_0417")
if "dam" in dms and (tasks_cfg.get("dam") or {}).get("type") == "multi":
old_dam = dms.pop("dam")
if old_dam.get("modes"):
dms["dam"] = old_dam
else:
legacy_dam["batch_0516"] = old_dam
dam_cfg = tasks_cfg.get("dam") or {}
if dam_cfg.get("type") == "multi" or legacy_dam:
_merge_multi_task_entry(dms, "dam", dam_cfg, legacy_dam)
return dms
def _build_catalog(wf: dict, *, prefer_reports: bool = True) -> tuple[dict[str, Any], str]:
out: dict[str, Any] = {"workspace": str(WORKSPACE), "dms": {}, "lane": {}}
build_source = "scan"
reports = load_dms_reports() if prefer_reports else None
reports = load_dms_reports()
report_splits: dict[tuple[str, str], dict[str, int]] = {}
report_classes: dict[str, dict[str, int]] = {}
if reports:
report_splits, report_classes = reports
build_source = "reports"
build_source = "reports" if prefer_reports else "scan+reports"
root = proj_root(wf, "dms")
reg_path = root / wf["projects"]["dms"]["registry"]
@@ -620,61 +836,91 @@ def _build_catalog(wf: dict, *, prefer_reports: bool = True) -> tuple[dict[str,
if summary_path.is_file():
class_by_task = _parse_class_summary(summary_path.read_text(encoding="utf-8"))
for task, tcfg in reg.get("tasks", {}).items():
entry: dict[str, Any] = {
"type": tcfg.get("type"),
"nc": tcfg.get("nc"),
"names": tcfg.get("names"),
"class_counts": class_by_task.get(task, {}),
"packs": [],
"drop_paths": {
"inbox": str((root / "inbox" / task).resolve()),
"sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "sources" / "<batch>").resolve()),
},
}
for p in packs_reg.get("packs", []):
pack_name = p["name"]
try:
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
except ValueError:
continue
task_data = pack_dir / tcfg.get("task_dir", task)
rep = report_splits.get((task, pack_name))
if rep:
def _pack_row(task_id: str, mode: str | None, mcfg: dict, pack_name: str, p: dict, task_data: Path) -> dict[str, Any]:
rep_key = report_task_key(task_id, mode)
rep = report_splits.get((rep_key, pack_name))
split_counts = _count_split_images(task_data)
scan_total = sum(split_counts.values())
if rep and (scan_total == 0 or not _task_data_has_labels(task_data)):
split_counts = {"train": rep["train"], "val": rep["val"], "test": rep["test"]}
class_counts = report_classes.get(rep_key, class_by_task.get(rep_key, {}))
bbox_points = _collect_bbox_points_sample(task_data) if _task_data_has_labels(task_data) else []
label_distribution = {
"class_counts": class_counts,
"label_files": 0,
"sampled": True,
"total_boxes": sum(class_counts.values()) if class_counts else 0,
"bbox_points": bbox_points,
}
else:
label_distribution = _collect_pack_label_distribution(task_data, mcfg)
if not label_distribution["class_counts"] and rep_key in report_classes:
label_distribution["class_counts"] = report_classes[rep_key]
if scan_total == 0 and rep:
split_counts = {"train": rep["train"], "val": rep["val"], "test": rep["test"]}
class_counts = report_classes.get(task, class_by_task.get(task, {}))
bbox_points = _collect_bbox_points_sample(task_data)
label_distribution = {
"class_counts": class_counts,
"label_files": 0,
"sampled": True,
"total_boxes": sum(class_counts.values()) if class_counts else 0,
"bbox_points": bbox_points,
}
else:
split_counts = _count_split_images(task_data)
label_distribution = _collect_pack_label_distribution(task_data, tcfg)
if not label_distribution["class_counts"] and task in report_classes:
label_distribution["class_counts"] = report_classes[task]
entry["packs"].append({
"name": pack_name,
"path": p.get("path"),
"role": p.get("role"),
"frozen": p.get("frozen", False),
"enabled": pack_name in wf["projects"]["dms"].get("active_packs", []),
"train_images": split_counts.get("train", 0),
"val_images": split_counts.get("val", 0),
"test_images": split_counts.get("test", 0),
"class_counts": label_distribution["class_counts"],
"label_files": label_distribution["label_files"],
"total_boxes": label_distribution["total_boxes"],
"sampled": label_distribution["sampled"],
"bbox_points": label_distribution["bbox_points"],
})
if not entry["class_counts"] and task in report_classes:
entry["class_counts"] = report_classes[task]
return {
"name": pack_name,
"path": p.get("path"),
"role": p.get("role"),
"frozen": p.get("frozen", False),
"enabled": pack_name in wf["projects"]["dms"].get("active_packs", []),
"train_images": split_counts.get("train", 0),
"val_images": split_counts.get("val", 0),
"test_images": split_counts.get("test", 0),
"class_counts": label_distribution["class_counts"],
"label_files": label_distribution["label_files"],
"total_boxes": label_distribution["total_boxes"],
"sampled": label_distribution["sampled"],
"bbox_points": label_distribution["bbox_points"],
}
for task, entry in iter_catalog_tasks(reg):
tcfg = reg["tasks"][task]
if tcfg.get("type") == "multi":
entry["drop_paths"] = {
**{
f"inbox_{mode}": str(inbox_dir(root, task, mode, reg).resolve())
for mode in (tcfg.get("modes") or {})
},
"sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "<subdir>" / "sources" / "<batch>").resolve()),
}
for mode in (tcfg.get("modes") or {}):
mcfg = get_mode_config(task, mode, reg)
mode_entry = entry["modes"][mode]
rep_key = report_task_key(task, mode)
mode_entry["class_counts"] = class_by_task.get("isa_detect", class_by_task.get(rep_key, {})) if mode == "detect" else class_by_task.get("isa_class_0116", class_by_task.get(rep_key, {}))
if not mode_entry["class_counts"] and rep_key in report_classes:
mode_entry["class_counts"] = report_classes[rep_key]
for p in packs_reg.get("packs", []):
pack_name = p["name"]
try:
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
except ValueError:
continue
task_data = resolve_task_data_for_scan(pack_dir, mcfg["task_dir"])
mode_entry["packs"].append(_pack_row(task, mode, mcfg, pack_name, p, task_data))
else:
entry["drop_paths"] = {
"inbox": str(inbox_dir(root, task, None, reg).resolve()),
"sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "sources" / "<batch>").resolve()),
}
rep_key = task
if not entry["class_counts"]:
entry["class_counts"] = class_by_task.get(task, report_classes.get(task, {}))
for p in packs_reg.get("packs", []):
pack_name = p["name"]
try:
pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
except ValueError:
continue
task_data = resolve_task_data_for_scan(pack_dir, tcfg.get("task_dir", task))
entry["packs"].append(_pack_row(task, None, tcfg, pack_name, p, task_data))
if not entry["class_counts"] and task in report_classes:
entry["class_counts"] = report_classes[task]
out["dms"][task] = entry
out["dms"] = _normalize_catalog_dms(out["dms"], reg)
root = proj_root(wf, "lane")
try:
reg = load_pack_registry("lane", root, wf)
@@ -718,6 +964,12 @@ def get_catalog(
full_catalog, build_source = _build_catalog(wf, prefer_reports=not refresh)
store_catalog_cache(sig, full_catalog, build_source=build_source)
cache_meta = {"cached": False, "build_source": build_source}
else:
reg_path = proj_root(wf, "dms") / wf["projects"]["dms"]["registry"]
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8")) if reg_path.is_file() else {}
dms = _normalize_catalog_dms(full_catalog.get("dms") or {}, reg)
if dms != full_catalog.get("dms"):
full_catalog = {**full_catalog, "dms": dms}
result: dict[str, Any]
if project == "dms" and task_or_pack:

View File

@@ -0,0 +1,91 @@
"""DMS inbox 原图批次(如 addw/ddaw仅有 images/,待送标,无 YOLO/COCO 标注)。"""
from __future__ import annotations
from pathlib import Path
from as_platform.data.batch import count_images, dms_has_images
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
def _count_images(path: Path) -> int:
return count_images(path)
def _split_image_counts(root: Path) -> dict[str, int]:
train = _count_images(root / "images" / "train")
val = _count_images(root / "images" / "val")
test = _count_images(root / "images" / "test")
if train + val + test == 0:
train = _count_images(root / "images")
if train + val + test == 0 and root.name == "images":
train = _count_images(root / "train")
val = _count_images(root / "val")
test = _count_images(root / "test")
if train + val + test == 0:
train = _count_images(root)
if train + val + test == 0 and (root / "train").is_dir():
train = _count_images(root / "train")
val = _count_images(root / "val")
test = _count_images(root / "test")
return {"train": train, "val": val, "test": test}
def _has_label_txts(root: Path) -> bool:
for sub in ("labels", "labels/train"):
d = root / sub
if d.is_dir() and any(d.rglob("*.txt")):
return True
return False
def _is_inbox_raw_layout(root: Path) -> bool:
if _has_label_txts(root):
return False
if dms_has_images(root):
return True
if root.name == "images" and (_count_images(root / "train") > 0 or _count_images(root) > 0):
return True
if (root / "train").is_dir() and _count_images(root / "train") > 0:
return True
if _count_images(root) > 0 and not (root / "images").is_dir() and not (root / "labels").is_dir():
return True
return False
class DmsInboxRawAdapter(IngestAdapter):
format_id = "dms_inbox_raw"
projects = ("dms",)
def can_handle(self, ctx: IngestContext) -> bool:
return _is_inbox_raw_layout(ctx.source_path)
def inspect(self, ctx: IngestContext) -> NormalizedDataset:
root = ctx.source_path
split_counts = _split_image_counts(root)
sample_count = sum(split_counts.values())
warnings: list[str] = []
if sample_count == 0:
warnings.append(
"未找到图片;请填批次根目录(含 images/train/)或 images/、train/ 目录"
)
artifacts: list[str] = []
if (root / "images").is_dir():
artifacts.append("images/")
elif (root / "train").is_dir():
artifacts.append("train/(将规范为 images/train")
elif root.name == "images":
artifacts.append("images/")
return NormalizedDataset(
format_id=self.format_id,
project=ctx.project,
task=ctx.task,
source_path=str(root),
split_counts=split_counts,
sample_count=sample_count,
annotation_count=0,
artifacts=artifacts,
warnings=warnings,
extra={"stage_hint": "raw_pool"},
)

View File

@@ -5,6 +5,7 @@ from pathlib import Path
from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
from as_platform.data.ingest.dms_coco import DmsCocoAdapter
from as_platform.data.ingest.dms_inbox_raw import DmsInboxRawAdapter
from as_platform.data.ingest.dms_yolo import DmsYoloAdapter
from as_platform.data.ingest.lane_lines import LaneLinesAdapter
from as_platform.data.ingest.lane_mask import LaneMaskAdapter
@@ -17,6 +18,7 @@ class UnknownFormatError(ValueError):
ADAPTERS: tuple[IngestAdapter, ...] = (
DmsYoloAdapter(),
DmsCocoAdapter(),
DmsInboxRawAdapter(),
LaneMaskAdapter(),
LaneLinesAdapter(),
)
@@ -32,9 +34,15 @@ def detect_adapter(ctx: IngestContext) -> IngestAdapter:
continue
if adapter.can_handle(ctx):
return adapter
hint = ""
if ctx.project == "dms":
hint = (
"DMS 送标/inbox 请使用批次根目录,且至少包含 images/train/*.jpg"
"(或已标注的 images/+labels/、COCO annotations/"
)
raise UnknownFormatError(
f"unable to detect format for project={ctx.project}, task={ctx.task}, "
f"source={ctx.source_path}. supported={available_formats(ctx.project)}"
f"source={ctx.source_path}. supported={available_formats(ctx.project)}{hint}"
)

View File

@@ -8,10 +8,14 @@ import uuid
import zipfile
from datetime import datetime
from pathlib import Path
from typing import BinaryIO
from typing import Any, BinaryIO
import yaml
from as_platform.config import MANIFESTS
from as_platform.data.batch import META_FILENAME, dms_has_images, enrich_batch, write_meta
from as_platform.data.catalog_cache import invalidate_catalog_cache
from as_platform.data.core import load_wf, proj_root
from as_platform.data.ingest import inspect_uploaded_dataset
from as_platform.db.engine import session_scope
from as_platform.db.models import DatasetCandidate
@@ -37,6 +41,7 @@ def create_uploaded_candidate(
*,
project: str,
task: str | None,
mode: str | None = None,
original_name: str,
upload_size_bytes: int,
submitted_by_name: str | None,
@@ -51,6 +56,7 @@ def create_uploaded_candidate(
id=candidate_id,
project=project,
task=task,
mode=mode,
status="uploaded",
source_type="upload",
original_name=original_name,
@@ -82,15 +88,17 @@ def write_candidate_upload(candidate_id: str, stream: BinaryIO, chunk_size: int
return str(path)
def list_candidates(limit: int = 100) -> list[dict]:
def list_candidates(*, offset: int = 0, limit: int = 20) -> dict[str, Any]:
with session_scope() as db:
rows = (
db.query(DatasetCandidate)
.order_by(DatasetCandidate.created_at.desc())
.limit(limit)
.all()
)
return [r.to_dict() for r in rows]
q = db.query(DatasetCandidate).order_by(DatasetCandidate.created_at.desc())
total = q.count()
rows = q.offset(max(0, offset)).limit(max(1, limit)).all()
return {
"items": [r.to_dict() for r in rows],
"total": total,
"offset": offset,
"limit": limit,
}
def get_candidate(candidate_id: str) -> dict | None:
@@ -153,6 +161,8 @@ def analyze_uploaded_candidate(candidate_id: str) -> dict:
try:
dataset_root = _extract_to_staging(upload_path, staging_dir)
if project == "dms":
dataset_root = _ensure_dms_inbox_layout(dataset_root)
normalized = inspect_uploaded_dataset(project, task, dataset_root)
report_file.parent.mkdir(parents=True, exist_ok=True)
report_file.write_text(json.dumps(normalized.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
@@ -177,3 +187,306 @@ def analyze_uploaded_candidate(candidate_id: str) -> dict:
rec.error_message = str(e)
db.flush()
raise
_SPLIT_DIR_NAMES = frozenset({"train", "val", "test"})
_STRUCTURE_DIR_NAMES = frozenset({"images", "labels", "annotations"})
def _dataset_root_from_dir(source_dir: Path) -> Path:
"""解析批次根目录;勿把 images/ 或 train/ 误剥成更深层导致丢失 images 层。"""
if not source_dir.is_dir():
raise FileNotFoundError(f"not a directory: {source_dir}")
subdirs = [p for p in source_dir.iterdir() if p.is_dir() and not p.name.startswith(".")]
files = [p for p in source_dir.iterdir() if p.is_file()]
if len(subdirs) == 1 and not files:
only = subdirs[0]
if only.name in _SPLIT_DIR_NAMES or only.name in _STRUCTURE_DIR_NAMES:
return source_dir
return only
return source_dir
def _ensure_dms_inbox_layout(root: Path) -> Path:
"""将 dataset/train 布局规范为 …/images/train已是批次根或 images/ 目录则不改动。"""
if not root.is_dir():
return root
from as_platform.data.batch import count_images, dms_has_images
if dms_has_images(root):
return root
if root.name == "images" and count_images(root / "train") > 0:
return root
if count_images(root) > 0 and not any(
(root / sub).is_dir() for sub in ("images", "train", "labels")
):
images_dir = root / "images"
images_dir.mkdir(parents=True, exist_ok=True)
dest_train = images_dir / "train"
dest_train.mkdir(parents=True, exist_ok=True)
for item in list(root.iterdir()):
if item.is_file() and item.suffix.lower() in {".jpg", ".jpeg", ".png", ".bmp", ".webp"}:
shutil.move(str(item), str(dest_train / item.name))
return root
train_dir = root / "train"
if not train_dir.is_dir() or count_images(train_dir) == 0:
return root
images_dir = root / "images"
if images_dir.is_dir() and count_images(images_dir / "train") > 0:
return root
images_dir.mkdir(parents=True, exist_ok=True)
dest_train = images_dir / "train"
if dest_train.exists():
shutil.rmtree(dest_train, ignore_errors=True)
shutil.move(str(train_dir), str(dest_train))
return root
def analyze_directory_candidate(candidate_id: str, source_dir: Path | None = None) -> dict:
"""分析目录型数据源(飞书 data_path / NAS无需 zip 上传。"""
upload_dir, staging_dir, report_file = _candidate_dirs(candidate_id)
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if not rec:
raise ValueError(f"candidate not found: {candidate_id}")
rec.status = "analyzing"
rec.error_message = None
db.flush()
project = rec.project
task = rec.task
root = source_dir or Path(rec.upload_path)
if not root.is_dir():
msg = f"source directory missing: {root}"
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if rec:
rec.status = "failed"
rec.error_message = msg
raise FileNotFoundError(msg)
try:
dataset_root = _dataset_root_from_dir(root)
if staging_dir.exists():
shutil.rmtree(staging_dir)
shutil.copytree(dataset_root, staging_dir / "dataset", dirs_exist_ok=True)
analyzed_root = staging_dir / "dataset"
if project == "dms":
analyzed_root = _ensure_dms_inbox_layout(analyzed_root)
normalized = inspect_uploaded_dataset(project, task, analyzed_root)
report_file.parent.mkdir(parents=True, exist_ok=True)
report_file.write_text(json.dumps(normalized.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if not rec:
raise ValueError(f"candidate not found during finalize: {candidate_id}")
rec.status = "analyzed"
rec.analyzed_source_path = str(analyzed_root)
rec.format_id = normalized.format_id
rec.set_split_counts(normalized.split_counts)
rec.set_quality(normalized.to_dict())
rec.error_message = None
db.flush()
invalidate_catalog_cache()
return normalized.to_dict()
except Exception as e:
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if rec:
rec.status = "failed"
rec.error_message = str(e)
db.flush()
raise
def create_directory_candidate(
*,
project: str,
task: str | None,
mode: str | None,
source_dir: Path,
source_type: str = "platform_delivery",
external_id: str | None = None,
feishu_record_id: str | None = None,
) -> dict:
"""为外部目录(平台送标申请 / 飞书台账)创建入湖候选。"""
candidate_id = _new_candidate_id()
upload_dir, _, _ = _candidate_dirs(candidate_id)
upload_dir.mkdir(parents=True, exist_ok=True)
with session_scope() as db:
rec = DatasetCandidate(
id=candidate_id,
project=project,
task=task,
mode=mode,
status="uploaded",
source_type=source_type,
original_name=source_dir.name,
upload_path=str(source_dir.resolve()),
upload_size_bytes=0,
external_id=external_id,
feishu_record_id=feishu_record_id,
)
db.add(rec)
db.flush()
return rec.to_dict()
def create_feishu_directory_candidate(
*,
project: str,
task: str | None,
mode: str | None,
source_dir: Path,
external_id: str | None = None,
feishu_record_id: str | None = None,
) -> dict:
"""为飞书台账行创建候选并指向 NAS/本地目录。"""
return create_directory_candidate(
project=project,
task=task,
mode=mode,
source_dir=source_dir,
source_type="feishu_bitable",
external_id=external_id,
feishu_record_id=feishu_record_id,
)
def _copy_tree_into(dest: Path, src: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for item in src.iterdir():
target = dest / item.name
if item.is_dir():
if target.exists():
_copy_tree_into(target, item)
else:
shutil.copytree(item, target)
else:
shutil.copy2(item, target)
def _resolve_dms_inbox_dest(root: Path, reg: dict, task: str, mode: str | None, batch_name: str) -> Path:
import sys
from as_platform.config import WORKSPACE
scripts = WORKSPACE / "datasets" / "dms" / "scripts"
if str(scripts) not in sys.path:
sys.path.insert(0, str(scripts))
from task_registry import inbox_dir, resolve_task_id
task_r, mode_r = resolve_task_id(task, mode)
tcfg = (reg.get("tasks") or {}).get(task_r) or {}
ib = inbox_dir(root, task_r, mode_r, reg)
if tcfg.get("type") == "multi" and mode_r:
if dms_has_images(ib) or (ib / META_FILENAME).is_file():
return ib
return ib / batch_name
return ib / batch_name
def promote_candidate_to_inbox(
candidate_id: str,
*,
batch: str | None = None,
mode: str | None = None,
) -> dict:
"""将 analyzed 候选数据复制到 inbox并登记 batch.meta。"""
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if not rec:
raise ValueError(f"candidate not found: {candidate_id}")
if rec.status not in ("analyzed",):
raise ValueError(f"candidate 状态须为 analyzed当前: {rec.status}")
if not rec.analyzed_source_path:
raise ValueError("缺少 analyzed_source_path请先完成分析")
project = rec.project
task = rec.task
eff_mode = mode or rec.mode
src = Path(rec.analyzed_source_path)
cand_format_id = rec.format_id
if not src.is_dir():
raise FileNotFoundError(f"分析目录不存在: {src}")
wf = load_wf()
root = proj_root(wf, project)
batch_name = batch or src.name or candidate_id.split("-", 1)[-1]
if project == "dms":
if not task:
raise ValueError("DMS 晋级需要 task")
reg_path = root / wf["projects"]["dms"]["registry"]
reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
tcfg = (reg.get("tasks") or {}).get(task) or {}
if tcfg.get("type") == "multi" and not eff_mode:
raise ValueError(f"任务 {task} 为 multi须指定 mode如 batch_0516")
dest = _resolve_dms_inbox_dest(root, reg, task, eff_mode, batch_name)
reg_batch = eff_mode or batch_name
else:
dest = root / "inbox" / batch_name
reg_batch = batch_name
if dest.exists() and any(dest.iterdir()):
_copy_tree_into(dest, src)
else:
if dest.exists():
shutil.rmtree(dest)
shutil.copytree(src, dest)
row = enrich_batch(
dest,
project=project,
task=task if project == "dms" else None,
pack=None,
batch=reg_batch,
location="inbox",
)
fmt = row.get("format", "yolo")
if cand_format_id == "dms_inbox_raw":
fmt = "inbox_raw"
meta_payload = {
"schema": "huaxu-batch-v1",
"project": project,
"task": task,
"batch": reg_batch,
"stage": "raw_pool",
"location": "inbox",
"format": fmt,
"counts": row.get("counts", {}),
}
if eff_mode:
meta_payload["mode"] = eff_mode
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if rec:
if rec.external_id:
meta_payload["external_id"] = rec.external_id
if rec.feishu_record_id:
meta_payload["feishu_record_id"] = rec.feishu_record_id
if rec.source_type and rec.source_type != "upload":
meta_payload["source_type"] = rec.source_type
write_meta(dest, meta_payload)
meta = {**row, "stage": "raw_pool"}
with session_scope() as db:
rec = db.get(DatasetCandidate, candidate_id)
if rec:
rec.status = "promoted"
rec.inbox_path = str(dest)
rec.promoted_batch = reg_batch
if eff_mode:
rec.mode = eff_mode
db.flush()
invalidate_catalog_cache()
return {
"ok": True,
"candidate_id": candidate_id,
"inbox_path": str(dest),
"batch": meta.get("batch", reg_batch),
"stage": meta.get("stage"),
"counts": meta.get("counts"),
}

View File

@@ -0,0 +1,179 @@
"""世界模型仿真数据生成 — API 接口层。
实际生成逻辑由外部世界模型引擎完成(通过 subprocess / HTTP 调用)。
本模块提供:任务提交、队列管理、状态追踪、结果入库。
"""
from __future__ import annotations
import json
import uuid
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from as_platform.config import WORKSPACE
from as_platform.db.engine import session_scope
SIM_JOBS_DIR = WORKSPACE / "manifests" / "simulation_jobs"
SIM_OUTPUT_ROOT = WORKSPACE / "datasets" / "dms" / "simulated"
SCENE_TEMPLATES = {
"urban_highway": {"label": "城市快速路", "desc": "多车道高速公路场景"},
"urban_street": {"label": "城市街道", "desc": "有交通灯和路口的城市道路"},
"rural_road": {"label": "乡村道路", "desc": "双车道乡村公路"},
"tunnel": {"label": "隧道", "desc": "隧道内光照变化场景"},
"night_city": {"label": "夜间城市", "desc": "低光照城市道路"},
"rain_highway": {"label": "雨天高速", "desc": "雨天湿滑路面"},
"fog_rural": {"label": "雾天乡村", "desc": "大雾低能见度"},
}
CAMERA_PRESETS = {
"truck_front": {"label": "卡车前视", "height": 2.5, "fov": 75, "pitch": -5},
"truck_side": {"label": "卡车侧视", "height": 2.5, "fov": 100, "pitch": 0},
"car_front": {"label": "轿车前视", "height": 1.2, "fov": 60, "pitch": -3},
"car_wide": {"label": "轿车广角", "height": 1.2, "fov": 120, "pitch": 0},
}
OBJECT_CLASSES = ["Pedestrain", "Car", "Truck", "Bus", "Motor-vehicles", "Tricycle", "cones"]
def list_jobs(offset: int = 0, limit: int = 20) -> dict[str, Any]:
SIM_JOBS_DIR.mkdir(parents=True, exist_ok=True)
jobs = []
for f in sorted(SIM_JOBS_DIR.glob("*.json"), reverse=True):
try:
data = json.loads(f.read_text())
data["_id"] = f.stem
if data.get("status") != "archived":
jobs.append(data)
except Exception:
pass
total = len(jobs)
return {"items": jobs[offset:offset + limit], "total": total}
def submit_job(params: dict[str, Any], user_name: str = "") -> dict[str, Any]:
job_id = f"sim-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
now = datetime.now(timezone.utc).isoformat()
scene = params.get("scene", "urban_highway")
camera = params.get("camera", "truck_front")
weather = params.get("weather", "clear")
objects = params.get("objects", OBJECT_CLASSES[:4])
density = params.get("density", "medium")
count = min(params.get("count", 100), 5000)
note = params.get("note", "")
fov_variant = params.get("fov_variant", False) # 是否生成多FOV变体
scene_info = SCENE_TEMPLATES.get(scene, {"label": scene})
cam_info = CAMERA_PRESETS.get(camera, {"label": camera})
job = {
"id": job_id,
"status": "queued",
"created_at": now,
"submitted_by": user_name,
"params": {
"scene": scene,
"scene_label": scene_info["label"],
"camera": camera,
"camera_label": cam_info["label"],
"camera_height": cam_info.get("height", 2.5),
"camera_fov": cam_info.get("fov", 75),
"weather": weather,
"objects": objects,
"density": density,
"count": count,
"fov_variant": fov_variant,
"note": note,
},
"result": None,
"batch_registered": False,
}
SIM_JOBS_DIR.mkdir(parents=True, exist_ok=True)
(SIM_JOBS_DIR / f"{job_id}.json").write_text(json.dumps(job, ensure_ascii=False, indent=2))
# Queue actual generation (mock for now — replace with real world model call)
_trigger_generation(job_id)
return job
def get_job(job_id: str) -> dict[str, Any] | None:
f = SIM_JOBS_DIR / f"{job_id}.json"
if not f.is_file():
return None
data = json.loads(f.read_text())
data["_id"] = f.stem
return data
def get_job_images(job_id: str, offset: int = 0, limit: int = 60) -> dict[str, Any]:
out_dir = SIM_OUTPUT_ROOT / job_id / "images"
if not out_dir.is_dir():
return {"items": [], "total": 0}
imgs = sorted(out_dir.glob("*.jpg")) + sorted(out_dir.glob("*.png"))
total = len(imgs)
page = imgs[offset:offset + limit]
return {
"items": [{"name": p.name, "path": str(p.relative_to(SIM_OUTPUT_ROOT))} for p in page],
"total": total,
}
def ingest_job_to_batch(job_id: str, task: str = "adas", user_name: str = "") -> dict[str, Any]:
"""将仿真生成的数据注册为批次,直接入库。"""
job = get_job(job_id)
if not job:
return {"ok": False, "error": "Job not found"}
out_dir = SIM_OUTPUT_ROOT / job_id
if not out_dir.is_dir():
return {"ok": False, "error": "生成数据不存在"}
# Register as batch
from as_platform.data.core import register_batch
batch_name = f"sim_{job_id}"
try:
register_batch(None, "dms", task, batch_name, stage="returned", location="inbox")
# Update job status
job["status"] = "ingested"
job["batch_registered"] = True
job["batch_name"] = batch_name
(SIM_JOBS_DIR / f"{job_id}.json").write_text(json.dumps(job, ensure_ascii=False, indent=2))
return {"ok": True, "batch": batch_name, "task": task}
except Exception as e:
return {"ok": False, "error": str(e)}
def _trigger_generation(job_id: str) -> None:
"""触发实际生成(当前为 mock。替换为真实世界模型调用。"""
import threading
def _run():
try:
# TODO: 替换为真实世界模型调用
# 例如: subprocess.run(["python", "world_model/generate.py", "--job", job_id])
_update_status(job_id, "running")
# Mock: create output directory but no actual images
out_dir = SIM_OUTPUT_ROOT / job_id / "images"
out_dir.mkdir(parents=True, exist_ok=True)
# Mock 完成
_update_status(job_id, "completed")
except Exception as e:
_update_status(job_id, "failed", str(e))
threading.Thread(target=_run, daemon=True, name=f"sim-{job_id}").start()
def _update_status(job_id: str, status: str, error: str = "") -> None:
f = SIM_JOBS_DIR / f"{job_id}.json"
if f.is_file():
data = json.loads(f.read_text())
data["status"] = status
if error:
data["error"] = error
if status == "completed":
data["completed_at"] = datetime.now(timezone.utc).isoformat()
f.write_text(json.dumps(data, ensure_ascii=False, indent=2))

View File

@@ -0,0 +1,161 @@
"""数据集版本管理 — snapshot / diff / lineage。版本以 JSON 文件存储在 datasets/<project>/versions/。"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from as_platform.config import WORKSPACE
from as_platform.data.core import get_catalog
def _versions_dir(project: str) -> Path:
return WORKSPACE / "datasets" / project / "versions"
def list_versions(project: str = "dms") -> list[dict[str, Any]]:
"""列出所有版本,按时间倒序。"""
vdir = _versions_dir(project)
if not vdir.is_dir():
return []
versions: list[dict[str, Any]] = []
for f in sorted(vdir.glob("*.json"), reverse=True):
try:
data = json.loads(f.read_text(encoding="utf-8"))
data["_id"] = f.stem
versions.append(data)
except Exception:
pass
return versions
def get_version(project: str, version_id: str) -> dict[str, Any] | None:
vf = _versions_dir(project) / f"{version_id}.json"
if not vf.is_file():
return None
data = json.loads(vf.read_text(encoding="utf-8"))
data["_id"] = vf.stem
return data
def create_snapshot(project: str, description: str = "", author: str = "") -> dict[str, Any]:
"""基于当前 catalog 创建数据集快照。"""
catalog = get_catalog(refresh=True)
dms = catalog.get("dms", {})
lane = catalog.get("lane", {})
# Collect pack summary from catalog
packs_summary: dict[str, dict[str, Any]] = {}
all_batches: list[str] = []
total_images = 0
total_labels = 0
if project == "dms":
for task_id, entry in dms.items():
entry_d = entry if isinstance(entry, dict) else {}
for p in entry_d.get("packs", []) or []:
pname = p.get("name", "")
if pname not in packs_summary:
packs_summary[pname] = {
"train_images": 0, "val_images": 0, "test_images": 0,
"class_counts": {}, "tasks": [],
}
s = packs_summary[pname]
s["train_images"] += p.get("train_images", 0) or 0
s["val_images"] += p.get("val_images", 0) or 0
s["test_images"] += p.get("test_images", 0) or 0
if task_id not in s["tasks"]:
s["tasks"].append(task_id)
total_images += (p.get("train_images", 0) or 0) + (p.get("val_images", 0) or 0)
total_labels += p.get("label_files", 0) or 0
# Collect batch names from pending report
from as_platform.data.core import get_pending_report
report = get_pending_report()
for b in report.get("batches", []) or []:
if isinstance(b, dict) and b.get("project") == "dms":
all_batches.append(b.get("batch", ""))
# Determine version number
existing = list_versions(project)
version_num = len(existing) + 1
version_id = f"v{version_num}"
# Get parent version
parent_id = existing[0]["_id"] if existing else None
# Compute diff if parent exists
diff: dict[str, Any] | None = None
if parent_id and existing:
parent = existing[0]
parent_packs = set((parent.get("packs") or {}).keys())
current_packs = set(packs_summary.keys())
parent_batches = set(parent.get("batches", []) or [])
current_batches = set(all_batches)
diff = {
"added_packs": sorted(current_packs - parent_packs),
"removed_packs": sorted(parent_packs - current_packs),
"added_batches": sorted(current_batches - parent_batches),
"removed_batches": sorted(parent_batches - current_batches),
}
snapshot = {
"version_id": version_id,
"project": project,
"created_at": datetime.now(timezone.utc).isoformat(),
"description": description,
"author": author,
"parent_version": parent_id,
"summary": {
"packs_count": len(packs_summary),
"total_images": total_images,
"total_labels": total_labels,
"batches_count": len(all_batches),
},
"packs": packs_summary,
"batches": sorted(all_batches),
"diff": diff,
}
# Save to file
vdir = _versions_dir(project)
vdir.mkdir(parents=True, exist_ok=True)
(vdir / f"{version_id}.json").write_text(
json.dumps(snapshot, ensure_ascii=False, indent=2, default=str),
encoding="utf-8",
)
snapshot["_id"] = version_id
return snapshot
def diff_versions(project: str, v1: str, v2: str) -> dict[str, Any]:
"""对比两个版本。"""
a = get_version(project, v1)
b = get_version(project, v2)
if not a or not b:
return {"error": "版本不存在"}
a_packs = set((a.get("packs") or {}).keys())
b_packs = set((b.get("packs") or {}).keys())
pack_changes: list[dict[str, Any]] = []
for pname in a_packs | b_packs:
pa = (a.get("packs") or {}).get(pname, {})
pb = (b.get("packs") or {}).get(pname, {})
if pa != pb:
pack_changes.append({
"pack": pname,
"v1": {"train": pa.get("train_images", 0), "val": pa.get("val_images", 0)},
"v2": {"train": pb.get("train_images", 0), "val": pb.get("val_images", 0)},
})
return {
"v1": {"id": v1, "created": a.get("created_at"), "total": a.get("summary", {}).get("total_images", 0)},
"v2": {"id": v2, "created": b.get("created_at"), "total": b.get("summary", {}).get("total_images", 0)},
"added_packs": sorted(b_packs - a_packs),
"removed_packs": sorted(a_packs - b_packs),
"pack_changes": pack_changes,
"image_delta": (b.get("summary", {}).get("total_images", 0) or 0) - (a.get("summary", {}).get("total_images", 0) or 0),
}