ADAS/HSAP
2
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: HSAP platform v2 — modular navigation, quality review, audit log, world model simulation

Browse Source 
Major changes:
- New frontend (platform/web/): Vite + React 18 + TypeScript + Tailwind
- 4-module navigation: 数据送标 / 模型管理 / 车队管理 / 系统管理
- Data catalog with charts (DMS/ADAS/Lane 3-tab view)
- Quality review workflow (标注质检): Good/Fine/Bad scoring with auto-advance
- Audit enhancements: batch operations, rejection categories, Feishu notifications
- Operation audit log (操作日志)
- World model simulation studio (仿真工坊)
- Dataset version management with snapshots and diff
- ADAS 7-class dataset integration (138K images organized + compressed)
- User management with Feishu integration and pagination
- CRUD/search/filter on all pages, card layout redesign
- PIL-optimized image overlay rendering
- Auto-snapshot on build, in_review workflow stage
- Removed embedded algorithm code (now in workspace)
This commit is contained in:
Chengfang Lu
2026-06-03 11:40:21 +08:00
parent 7c43b44c57
commit e72bc061c5
5487 changed files with 979207 additions and 6197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
datasets/dms/configs/keypoint_order/addw_face_37.yaml Normal file
View File

@@ -0,0 +1,43 @@
# addw_face：37 点顺序与 Label Studio dms_pose.xml 中 kp_XX 标签一一对应
task: addw_face
kpt_shape: [37, 3]
class_names:
0: face
keypoints:
- {id: 0, label: kp_00}
- {id: 1, label: kp_01}
- {id: 2, label: kp_02}
- {id: 3, label: kp_03}
- {id: 4, label: kp_04}
- {id: 5, label: kp_05}
- {id: 6, label: kp_06}
- {id: 7, label: kp_07}
- {id: 8, label: kp_08}
- {id: 9, label: kp_09}
- {id: 10, label: kp_10}
- {id: 11, label: kp_11}
- {id: 12, label: kp_12}
- {id: 13, label: kp_13}
- {id: 14, label: kp_14}
- {id: 15, label: kp_15}
- {id: 16, label: kp_16}
- {id: 17, label: kp_17}
- {id: 18, label: kp_18}
- {id: 19, label: kp_19}
- {id: 20, label: kp_20}
- {id: 21, label: kp_21}
- {id: 22, label: kp_22}
- {id: 23, label: kp_23}
- {id: 24, label: kp_24}
- {id: 25, label: kp_25}
- {id: 26, label: kp_26}
- {id: 27, label: kp_27}
- {id: 28, label: kp_28}
- {id: 29, label: kp_29}
- {id: 30, label: kp_30}
- {id: 31, label: kp_31}
- {id: 32, label: kp_32}
- {id: 33, label: kp_33}
- {id: 34, label: kp_34}
- {id: 35, label: kp_35}
- {id: 36, label: kp_36}

7
datasets/dms/configs/label_studio/dam_15cls.xml Normal file
View File

@@ -0,0 +1,7 @@
<View>
<Image name="image" value="$image"/>
<RectangleLabels name="label" toName="image">
<Label value="face" category="0"/>
<Label value="driver" category="14"/>
</RectangleLabels>
</View>

9
datasets/dms/configs/label_studio/dms_detect.xml Normal file
View File

@@ -0,0 +1,9 @@
<View>
<Image name="image" value="$image"/>
<RectangleLabels name="label" toName="image">
<Label value="face"/>
<Label value="eye_open"/>
<Label value="nod_eye"/>
<Label value="nod_face"/>
</RectangleLabels>
</View>

46
datasets/dms/configs/label_studio/dms_pose.xml Normal file
View File

@@ -0,0 +1,46 @@
<View>
<Image name="image" value="$image" zoom="true"/>
<RectangleLabels name="bbox" toName="image">
<Label value="face"/>
</RectangleLabels>
<KeyPointLabels name="kpts" toName="image" choice="single">
<Label value="kp_00" maxUsages="1"/>
<Label value="kp_01" maxUsages="1"/>
<Label value="kp_02" maxUsages="1"/>
<Label value="kp_03" maxUsages="1"/>
<Label value="kp_04" maxUsages="1"/>
<Label value="kp_05" maxUsages="1"/>
<Label value="kp_06" maxUsages="1"/>
<Label value="kp_07" maxUsages="1"/>
<Label value="kp_08" maxUsages="1"/>
<Label value="kp_09" maxUsages="1"/>
<Label value="kp_10" maxUsages="1"/>
<Label value="kp_11" maxUsages="1"/>
<Label value="kp_12" maxUsages="1"/>
<Label value="kp_13" maxUsages="1"/>
<Label value="kp_14" maxUsages="1"/>
<Label value="kp_15" maxUsages="1"/>
<Label value="kp_16" maxUsages="1"/>
<Label value="kp_17" maxUsages="1"/>
<Label value="kp_18" maxUsages="1"/>
<Label value="kp_19" maxUsages="1"/>
<Label value="kp_20" maxUsages="1"/>
<Label value="kp_21" maxUsages="1"/>
<Label value="kp_22" maxUsages="1"/>
<Label value="kp_23" maxUsages="1"/>
<Label value="kp_24" maxUsages="1"/>
<Label value="kp_25" maxUsages="1"/>
<Label value="kp_26" maxUsages="1"/>
<Label value="kp_27" maxUsages="1"/>
<Label value="kp_28" maxUsages="1"/>
<Label value="kp_29" maxUsages="1"/>
<Label value="kp_30" maxUsages="1"/>
<Label value="kp_31" maxUsages="1"/>
<Label value="kp_32" maxUsages="1"/>
<Label value="kp_33" maxUsages="1"/>
<Label value="kp_34" maxUsages="1"/>
<Label value="kp_35" maxUsages="1"/>
<Label value="kp_36" maxUsages="1"/>
</KeyPointLabels>
</View>

23
datasets/dms/data_packs.yaml
View File

@@ -1,16 +1,15 @@
# DMS 数据包登记（训练用哪些包由 ML/workflow.registry.yaml 的 active_packs 决定）
schema: dms-multi-pack-v1
packs:
- name: dms_v1
path: packs/dms_v1
role: baseline
frozen: true
# 新建增量包示例（add/build 写入，需 enable 后才参与训练）:
# - name: dms_v2
# path: packs/dms_v2
- name: dms_v1
path: packs/dms_v1
role: baseline
frozen: true
- name: dms_v2
path: packs/dms_v2
role: incremental
- name: adas_v1
path: packs/adas_v1
role: adas_forward
note: 前向 ADAS 7 类检测数据集 (53GB)
aliases:
DMS_v1_AddBy_Luchengfang_20260520: dms_v1

85
datasets/dms/datasets.registry.yaml
View File

@@ -1,6 +1,7 @@
# 任务定义 + 训练超参。数据包路径见 data_packs.yaml；训练用哪些包见 ML/workflow.registry.yaml active_packs
# 任务定义 + 训练超参。数据包路径见 data_packs.yaml；训练用哪些包见 workflow.registry.yaml active_packs
# domain: dms=舱内驾驶员监控 | forward=前向交通标志（检测+细分类合一任务）
version: 2
version: 3
dataset_root: .
split:
@@ -47,6 +48,8 @@ train:
tasks:
ddaw:
domain: dms
label: DDAW 疲劳检测
type: detect
task_dir: ddaw
nc: 9
@@ -55,6 +58,8 @@ tasks:
extra_train: []
addw:
domain: dms
label: ADDW 分心检测
type: detect
task_dir: addw
nc: 4
@@ -63,6 +68,8 @@ tasks:
extra_train: []
addw_face:
domain: dms
label: ADDW 人脸关键点
type: pose
task_dir: addw_face
kpt_shape: [37, 3]
@@ -70,32 +77,56 @@ tasks:
inbox: inbox/addw_face
extra_train: []
isa:
type: detect
task_dir: isa
nc: 4
names: [indicative, prohibitory, warning, vehicle]
inbox: inbox/isa
extra_train: []
dam:
type: detect
domain: dms
label: DAM 驾驶员监控
type: multi
task_dir: dam
nc: 15
names: [face, eye_open, eye_partially_open, eye_close, mouth_open, mouth_partially_open, mouth_close, side_face, nod_face, glasses, sunglasses, smoke, phone, driver, rise_face]
inbox: inbox/dam
extra_train: []
modes:
batch_0516:
label: 0516 批次
type: detect
subdir: batch_0516
nc: 15
names: [face, eye_open, eye_partially_open, eye_close, mouth_open, mouth_partially_open, mouth_close, side_face, nod_face, glasses, sunglasses, smoke, phone, driver, rise_face]
inbox: inbox/dam/batch_0516
extra_train: []
batch_0417:
label: 0417 批次
type: detect
subdir: batch_0417
nc: 15
names: [face, eye_open, eye_partially_open, eye_close, mouth_open, mouth_partially_open, mouth_close, side_face, nod_face, glasses, sunglasses, smoke, phone, driver, rise_face]
inbox: inbox/dam/batch_0417
extra_train: []
dam_0417:
forward:
domain: forward
label: 前向·交通标志
type: multi
task_dir: forward
modes:
detect:
label: 粗检测（4 类）
type: detect
subdir: detect
nc: 4
names: [indicative, prohibitory, warning, vehicle]
inbox: inbox/forward/detect
extra_train: []
classify:
label: 细分类
type: classify
subdir: classify
inbox: inbox/forward/classify
extra_train: []
adas:
domain: forward
label: 前向·ADAS 7 类检测
type: detect
task_dir: dam_0417
nc: 15
names: [face, eye_open, eye_partially_open, eye_close, mouth_open, mouth_partially_open, mouth_close, side_face, nod_face, glasses, sunglasses, smoke, phone, driver, rise_face]
inbox: inbox/dam_0417
extra_train: []
isa_class:
type: classify
task_dir: isa_class
inbox: inbox/isa_class
extra_train: []
task_dir: adas
nc: 7
names: [Pedestrain, Car, Truck, Bus, Motor-vehicles, Tricycle, cones]
mode: detect
inbox: inbox/adas

215
datasets/dms/manifests/dataset_class_summary.txt
View File

@@ -1,207 +1,8 @@
======================================================================
【ddaw_1124】YOLO 检测 path: /home/chengfanglu/DATA/DMS/DATASET/gyp/ddaw_1124
======================================================================
当前划分: train=3824 val=424 test=0 合计=4248
类别数(nc): 9 类别ID: [0, 1, 2, 3, 4, 5, 6, 7, 8]
--- 当前：含该类别的图片数（按 split）---
cls train val val% test 合计
0 3072 348 10.2% 0 3420
1 686 65 8.7% 0 751
2 1827 210 10.3% 0 2037
3 619 71 10.3% 0 690
4 279 29 9.4% 0 308
5 881 99 10.1% 0 980
6 2415 273 10.2% 0 2688
7 659 64 8.9% 0 723
8 138 15 9.8% 0 153
当前 val 占比: min=8.7% max=10.3% 极差=1.7% (目标≈10%)
✓ 当前各类 val 比例已较均衡
--- 分层重分后(模拟, val_ratio=0.1, seed=42) ---
train=3823 val=425
cls train val val%
0 3074 346 10.1%
1 674 77 10.3%
2 1834 203 10.0%
3 620 70 10.1%
4 277 31 10.1%
5 882 98 10.0%
6 2423 265 9.9%
7 651 72 10.0%
8 138 15 9.8%
重分后 val 占比: min=9.8% max=10.3% 极差=0.4%
✓ 分层后各类 val 比例均衡（极差≤2%）
======================================================================
【addw_0523】YOLO 检测 path: /home/chengfanglu/DATA/DMS/DATASET/gyp/addw_0523
======================================================================
当前划分: train=2375 val=594 test=0 合计=2969
类别数(nc): 4 类别ID: [0, 1, 2, 3]
--- 当前：含该类别的图片数（按 split）---
cls train val val% test 合计
0 1367 370 21.3% 0 1737
1 415 120 22.4% 0 535
2 1018 229 18.4% 0 1247
3 1812 445 19.7% 0 2257
当前 val 占比: min=18.4% max=22.4% 极差=4.1% (目标≈10%)
⚠ 当前各类 val 比例不均衡
--- 分层重分后(模拟, val_ratio=0.1, seed=42) ---
train=2671 val=298
cls train val val%
0 1565 172 9.9%
1 481 54 10.1%
2 1121 126 10.1%
3 2029 228 10.1%
重分后 val 占比: min=9.9% max=10.1% 极差=0.2%
✓ 分层后各类 val 比例均衡（极差≤2%）
======================================================================
【isa_detect】YOLO 检测 path: /home/chengfanglu/DATA/DMS/DATASET/gyp/isa_detect
======================================================================
当前划分: train=50480 val=9584 test=0 合计=60064
类别数(nc): 4 类别ID: [0, 1, 2, 3]
--- 当前：含该类别的图片数（按 split）---
cls train val val% test 合计
0 15871 2971 15.8% 0 18842
1 33355 7060 17.5% 0 40415
2 7348 1204 14.1% 0 8552
3 14715 2429 14.2% 0 17144
当前 val 占比: min=14.1% max=17.5% 极差=3.4% (目标≈10%)
⚠ 当前各类 val 比例不均衡
--- 分层重分后(模拟, val_ratio=0.1, seed=42) ---
train=54066 val=5940
cls train val val%
0 16883 1902 10.1%
1 36344 4013 9.9%
2 7693 855 10.0%
3 15435 1707 10.0%
重分后 val 占比: min=9.9% max=10.1% 极差=0.2%
✓ 分层后各类 val 比例均衡（极差≤2%）
======================================================================
【dam_0516】YOLO 检测 path: /home/chengfanglu/DATA/DMS/DATASET/gyp/dam_0516
======================================================================
当前划分: train=3780 val=1890 test=0 合计=5670
类别数(nc): 15 类别ID: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
--- 当前：含该类别的图片数（按 split）---
cls train val val% test 合计
0 2582 1337 34.1% 0 3919
1 726 373 33.9% 0 1099
2 1670 829 33.2% 0 2499
3 784 395 33.5% 0 1179
4 266 136 33.8% 0 402
5 1141 542 32.2% 0 1683
6 2066 1050 33.7% 0 3116
7 670 324 32.6% 0 994
8 445 217 32.8% 0 662
9 1088 528 32.7% 0 1616
10 306 164 34.9% 0 470
11 838 464 35.6% 0 1302
12 883 489 35.6% 0 1372
13 3700 1843 33.2% 0 5543
14 360 151 29.5% 0 511
当前 val 占比: min=29.5% max=35.6% 极差=6.1% (目标≈10%)
⚠ 当前各类 val 比例不均衡
--- 分层重分后(模拟, val_ratio=0.1, seed=42) ---
train=5104 val=566
cls train val val%
0 3525 394 10.1%
1 986 113 10.3%
2 2240 259 10.4%
3 1074 105 8.9%
4 362 40 10.0%
5 1497 186 11.1%
6 2826 290 9.3%
7 896 98 9.9%
8 595 67 10.1%
9 1473 143 8.8%
10 423 47 10.0%
11 1180 122 9.4%
12 1241 131 9.5%
13 4988 555 10.0%
14 459 52 10.2%
重分后 val 占比: min=8.8% max=11.1% 极差=2.2%
~ 分层后明显改善，少数类因样本极少会有小幅偏差
======================================================================
【yoloface-0726】YOLO 检测 path: /home/chengfanglu/DATA/DMS/DATASET/gyp/yoloface-0726
======================================================================
当前划分: train=4145 val=1464 test=0 合计=5609
类别数(nc): 1 类别ID: [0]
--- 当前：含该类别的图片数（按 split）---
cls train val val% test 合计
0 4137 1462 26.1% 0 5599
当前 val 占比: min=26.1% max=26.1% 极差=0.0% (目标≈10%)
✓ 当前各类 val 比例已较均衡
--- 分层重分后(模拟, val_ratio=0.1, seed=42) ---
train=4970 val=551
cls train val val%
0 4960 551 10.0%
重分后 val 占比: min=10.0% max=10.0% 极差=0.0%
✓ 分层后各类 val 比例均衡（极差≤2%）
======================================================================
【isa_class_0116】分类(文件夹) path: /home/chengfanglu/DATA/DMS/DATASET/gyp/isa_class_0116
======================================================================
split: ['test', 'train'] 类别数: 92 总图片: 61747
--- 每类图片数（前20类 + 汇总）---
class test train
005 43 393
010 55 501
015 47 424
020 67 604
030 214 1933
040 152 1369
050 279 2513
060 224 2024
070 179 1614
080 285 2572
090 116 1045
100 207 1866
110 88 794
120 156 1407
130 47 429
arrow_right 126 1138
arrow_up 38 350
bycycle 40 368
dis_005 30 279
dis_010 23 213
(仅列前20类，共92类)
当前为 train/test 结构（无 val）；若需 val 请用 stratified_split.py classify --src-split train
若从 train 按类划 10% val: val% min=8.3% max=11.8% 极差=3.4%
test: 92类 min=1 max=310 avg=67 合计=6133
train: 92类 min=17 max=2792 avg=604 合计=55614
======================================================================
【dam_src_0417】原始 jpg+xml path: /home/chengfanglu/DATA/DMS/DATASET/gyp/dam_src_0417
======================================================================
src_data_0417_pick: jpg=2455 xml=2455 (无 YOLO 标签，待转换后按类划分)
[adas]
Pedestrain: 19730
Car: 241764
Truck: 87538
Bus: 6804
Motor-vehicles: 38378
Tricycle: 5654
cones: 6419

30
datasets/dms/manifests/train_versions.yaml
View File

@@ -1,7 +1,7 @@
# 模型版本：current=线上/基线，candidate=待晋级（train → eval → promote）
# data_yaml 由 active_packs 生成，路径固定为 manifests/yaml_active/
# 模型版本：current=线上/基线，candidate=待晋级
# multi 任务 forward 按 mode 分子项；训练 yaml 为 forward__detect / forward__classify
schema: model-versions-v2
schema: model-versions-v3
ddaw:
type: detect
@@ -24,30 +24,38 @@ addw_face:
candidate: null
last_eval: null
isa:
dam__batch_0516:
task: dam
mode: batch_0516
type: detect
data_yaml: manifests/yaml_active/isa.yaml
data_yaml: manifests/yaml_active/dam__batch_0516.yaml
current: null
candidate: null
last_eval: null
dam:
dam__batch_0417:
task: dam
mode: batch_0417
type: detect
data_yaml: manifests/yaml_active/dam.yaml
data_yaml: manifests/yaml_active/dam__batch_0417.yaml
current: null
candidate: null
last_eval: null
dam_0417:
forward__detect:
task: forward
mode: detect
type: detect
data_yaml: manifests/yaml_active/dam_0417.yaml
data_yaml: manifests/yaml_active/forward__detect.yaml
current: null
candidate: null
last_eval: null
isa_class:
forward__classify:
task: forward
mode: classify
type: classify
data_yaml: manifests/yaml_active/isa_class.yaml
data_yaml: manifests/yaml_active/forward__classify.yaml
current: null
candidate: null
last_eval: null

7
datasets/dms/manifests/yaml_active/dam__batch_0417.yaml Normal file
View File

@@ -0,0 +1,7 @@
# dam__batch_0417 — packs: dms_v1
path: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/dam/batch_0417
train: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/dam_0417/images/train
val: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/dam_0417/images/val
nc: 15
names: ["face", "eye_open", "eye_partially_open", "eye_close", "mouth_open", "mouth_partially_open", "mouth_close", "side_face", "nod_face", "glasses", "sunglasses", "smoke", "phone", "driver", "rise_face"]

7
datasets/dms/manifests/yaml_active/dam__batch_0516.yaml Normal file
View File

@@ -0,0 +1,7 @@
# dam__batch_0516 — packs: dms_v1
path: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/dam/batch_0516
train: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/_dam_stash_0516/images/train
val: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/_dam_stash_0516/images/val
nc: 15
names: ["face", "eye_open", "eye_partially_open", "eye_close", "mouth_open", "mouth_partially_open", "mouth_close", "side_face", "nod_face", "glasses", "sunglasses", "smoke", "phone", "driver", "rise_face"]

5
datasets/dms/manifests/yaml_active/forward__classify.yaml Normal file
View File

@@ -0,0 +1,5 @@
# forward__classify — pack: dms_v1
path: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/isa_class
train: train
val: val
test: test

7
datasets/dms/manifests/yaml_active/forward__detect.yaml Normal file
View File

@@ -0,0 +1,7 @@
# forward__detect — packs: dms_v1
path: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/forward/detect
train: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/isa/images/train
val: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/isa/images/val
nc: 4
names: ["indicative", "prohibitory", "warning", "vehicle"]

347
datasets/dms/scripts/export_ls_to_yolo.py Normal file
View File

@@ -0,0 +1,347 @@
#!/usr/bin/env python3
"""Label Studio ls_annotations JSON → YOLO detect / YOLO pose txt."""
from __future__ import annotations
import argparse
import hashlib
import json
import sys
from pathlib import Path
from typing import Any
import yaml
SCRIPT_DIR = Path(__file__).resolve().parent
DMS_ROOT = SCRIPT_DIR.parent
CONFIG_DIR = DMS_ROOT / "configs"
REGISTRY_PATH = DMS_ROOT / "datasets.registry.yaml"
KPT_ORDER_DIR = CONFIG_DIR / "keypoint_order"
IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".JPG", ".JPEG", ".PNG"}
ANNOTATIONS_DIRNAME = "ls_annotations"
def _load_registry() -> dict[str, Any]:
return yaml.safe_load(REGISTRY_PATH.read_text(encoding="utf-8"))
def _resolve_task_config(task: str, mode: str | None = None) -> dict[str, Any]:
from task_registry import get_mode_config, resolve_task_id
reg = _load_registry()
task_r, mode_r = resolve_task_id(task, mode)
return get_mode_config(task_r, mode_r, reg)
def _class_name_to_id(names: list[str] | dict[int | str, str]) -> dict[str, int]:
if isinstance(names, dict):
return {str(v): int(k) for k, v in names.items()}
return {name: idx for idx, name in enumerate(names)}
def _load_kpt_label_map(task: str) -> dict[str, int]:
path = KPT_ORDER_DIR / f"{task}_37.yaml"
if not path.is_file():
path = KPT_ORDER_DIR / f"{task}.yaml"
if not path.is_file():
raise FileNotFoundError(f"keypoint manifest not found for task {task}")
data = yaml.safe_load(path.read_text(encoding="utf-8"))
out: dict[str, int] = {}
for item in data.get("keypoints") or []:
out[str(item["label"])] = int(item["id"])
return out
def _task_id_for_image(image_path: Path, batch_dir: Path) -> str:
try:
rel = image_path.relative_to(batch_dir)
stem = rel.as_posix()
except ValueError:
stem = image_path.stem
return hashlib.sha256(stem.encode()).hexdigest()[:16]
def _iter_batch_images(batch_dir: Path) -> list[Path]:
if not batch_dir.is_dir():
return []
candidates: list[Path] = []
search_roots = [
batch_dir / "images",
batch_dir / "images" / "train",
batch_dir,
]
seen: set[str] = set()
for root in search_roots:
if not root.is_dir():
continue
for p in sorted(root.rglob("*")):
if not p.is_file() or p.suffix not in IMG_EXTS:
continue
key = str(p.resolve())
if key in seen:
continue
seen.add(key)
candidates.append(p.resolve())
return candidates
def _label_out_path(image_path: Path, batch_dir: Path) -> Path:
try:
rel = image_path.relative_to(batch_dir)
except ValueError:
rel = Path(image_path.name)
parts = list(rel.parts)
if parts and parts[0] == "images":
parts = parts[1:]
if parts and parts[0] in ("train", "val", "test"):
split = parts[0]
name = Path(*parts[1:]).with_suffix(".txt")
return batch_dir / "labels" / split / name
name = Path(*parts).with_suffix(".txt")
return batch_dir / "labels" / name
def _extract_result_regions(data: dict[str, Any]) -> list[dict[str, Any]]:
result = data.get("result")
if isinstance(result, list) and result:
return result
annotations = data.get("annotations")
if isinstance(annotations, list) and annotations:
first = annotations[0]
if isinstance(first, dict) and isinstance(first.get("result"), list):
return first["result"]
return []
def _clamp01(v: float) -> float:
return max(0.0, min(1.0, v))
def _ls_rect_to_yolo_bbox(value: dict[str, Any]) -> tuple[float, float, float, float]:
x = float(value["x"])
y = float(value["y"])
w = float(value["width"])
h = float(value["height"])
cx = _clamp01((x + w / 2.0) / 100.0)
cy = _clamp01((y + h / 2.0) / 100.0)
nw = _clamp01(w / 100.0)
nh = _clamp01(h / 100.0)
return cx, cy, nw, nh
def _ls_point_to_yolo_xy(value: dict[str, Any]) -> tuple[float, float]:
return _clamp01(float(value["x"]) / 100.0), _clamp01(float(value["y"]) / 100.0)
def _bbox_center(bbox: tuple[float, float, float, float]) -> tuple[float, float]:
cx, cy, _, _ = bbox
return cx, cy
def _parse_rectangles(
regions: list[dict[str, Any]],
class_map: dict[str, int],
) -> list[dict[str, Any]]:
boxes: list[dict[str, Any]] = []
for region in regions:
if region.get("type") != "rectanglelabels":
continue
value = region.get("value") or {}
labels = value.get("rectanglelabels") or []
if not labels:
continue
label = str(labels[0])
if label not in class_map:
continue
bbox = _ls_rect_to_yolo_bbox(value)
boxes.append({"class_id": class_map[label], "bbox": bbox, "region_id": region.get("id")})
return boxes
def _parse_keypoints(
regions: list[dict[str, Any]],
kpt_map: dict[str, int],
) -> list[dict[str, Any]]:
points: list[dict[str, Any]] = []
for region in regions:
rtype = region.get("type")
if rtype not in ("keypointlabels", "keypoint"):
continue
value = region.get("value") or {}
labels = value.get("keypointlabels") or []
if not labels:
continue
label = str(labels[0])
if label not in kpt_map:
continue
x, y = _ls_point_to_yolo_xy(value)
points.append({"index": kpt_map[label], "x": x, "y": y, "region_id": region.get("id")})
return points
def _assign_keypoints_to_boxes(
boxes: list[dict[str, Any]],
points: list[dict[str, Any]],
) -> dict[int | None, list[dict[str, Any]]]:
if not boxes:
return {None: points}
if len(boxes) == 1:
return {0: points}
assigned: dict[int, list[dict[str, Any]]] = {i: [] for i in range(len(boxes))}
for pt in points:
best_i = 0
best_d = float("inf")
for i, box in enumerate(boxes):
cx, cy = _bbox_center(box["bbox"])
d = (pt["x"] - cx) ** 2 + (pt["y"] - cy) ** 2
if d < best_d:
best_d = d
best_i = i
assigned[best_i].append(pt)
return assigned
def _format_detect_line(class_id: int, bbox: tuple[float, float, float, float]) -> str:
cx, cy, w, h = bbox
return f"{class_id} {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}"
def _format_pose_line(
class_id: int,
bbox: tuple[float, float, float, float],
points: list[dict[str, Any]],
nk: int,
) -> str:
slots: list[tuple[float, float, float]] = [(0.0, 0.0, 0.0)] * nk
for pt in points:
idx = int(pt["index"])
if 0 <= idx < nk:
slots[idx] = (pt["x"], pt["y"], 2.0)
parts = _format_detect_line(class_id, bbox).split()
for x, y, v in slots:
parts.extend([f"{x:.6f}", f"{y:.6f}", f"{v:.6f}"])
return " ".join(parts)
def convert_regions_to_yolo_lines(
regions: list[dict[str, Any]],
*,
mode: str,
class_map: dict[str, int],
kpt_map: dict[str, int] | None = None,
kpt_shape: list[int] | None = None,
) -> list[str]:
if mode == "detect":
lines = []
for box in _parse_rectangles(regions, class_map):
lines.append(_format_detect_line(box["class_id"], box["bbox"]))
return lines
if mode != "pose":
raise ValueError(f"unsupported mode: {mode}")
if not kpt_map or not kpt_shape:
raise ValueError("pose mode requires kpt_map and kpt_shape")
nk = int(kpt_shape[0])
boxes = _parse_rectangles(regions, class_map)
points = _parse_keypoints(regions, kpt_map)
if not boxes:
return []
assigned = _assign_keypoints_to_boxes(boxes, points)
lines: list[str] = []
for i, box in enumerate(boxes):
pts = assigned.get(i, [])
lines.append(_format_pose_line(box["class_id"], box["bbox"], pts, nk))
return lines
def export_batch(
batch_dir: Path,
task: str,
*,
mode: str,
task_mode: str | None = None,
out_subdir: str | None = None,
) -> dict[str, Any]:
batch_dir = batch_dir.resolve()
tcfg = _resolve_task_config(task, task_mode)
class_map = _class_name_to_id(tcfg.get("names") or {})
kpt_map: dict[str, int] | None = None
kpt_shape: list[int] | None = None
if mode == "pose":
kpt_map = _load_kpt_label_map(task)
kpt_shape = list(tcfg.get("kpt_shape") or [37, 3])
ann_dir = batch_dir / "labels" / ANNOTATIONS_DIRNAME
written = 0
skipped_empty = 0
missing_ann = 0
for image_path in _iter_batch_images(batch_dir):
task_id = _task_id_for_image(image_path, batch_dir)
ann_path = ann_dir / f"{task_id}.json"
if not ann_path.is_file():
missing_ann += 1
continue
data = json.loads(ann_path.read_text(encoding="utf-8"))
regions = _extract_result_regions(data)
if not regions:
skipped_empty += 1
continue
lines = convert_regions_to_yolo_lines(
regions,
mode=mode,
class_map=class_map,
kpt_map=kpt_map,
kpt_shape=kpt_shape,
)
if not lines:
skipped_empty += 1
continue
out_path = _label_out_path(image_path, batch_dir)
if out_subdir:
# 显式覆盖：相对 batch_dir 的子目录 + 文件名
out_path = batch_dir / out_subdir / f"{image_path.stem}.txt"
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
written += 1
return {
"ok": True,
"batch_dir": str(batch_dir),
"task": task,
"mode": mode,
"written": written,
"skipped_empty": skipped_empty,
"missing_ann": missing_ann,
"out_subdir": out_subdir or "auto",
}
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Export Label Studio annotations to YOLO txt")
parser.add_argument("--batch-dir", type=Path, required=True)
parser.add_argument("--task", required=True)
parser.add_argument("--mode", choices=("detect", "pose"), required=True)
parser.add_argument("--task-mode", default=None, help="dam batch_0516 / batch_0417 等")
parser.add_argument("--out-subdir", default="labels/train")
args = parser.parse_args(argv)
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
result = export_batch(
args.batch_dir,
args.task,
mode=args.mode,
task_mode=args.task_mode,
out_subdir=args.out_subdir,
)
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0 if result["written"] > 0 else 1
if __name__ == "__main__":
raise SystemExit(main())

51
datasets/dms/scripts/ingest_incremental.py
View File

@@ -236,7 +236,7 @@ def ingest_yolo(
if is_voc:
if tcfg["type"] != "detect":
raise SystemExit("VOC xml 仅支持 detect 任务（dam / dam_0417）")
raise SystemExit("VOC xml 仅支持 detect 任务（dam 各批次等）")
staging = data_root.parent / "_staging_voc" / task
staging_parent = staging.parent
if staging.exists() and not dry_run:
@@ -384,7 +384,11 @@ def ingest_one(
src: Path,
args: argparse.Namespace,
) -> dict:
tcfg = reg["tasks"][task]
from task_registry import get_mode_config, resolve_task_id
submode = getattr(args, "mode", None) or getattr(args, "submode", None)
task, submode = resolve_task_id(task, submode)
tcfg = get_mode_config(task, submode, reg)
pack = getattr(args, "pack", None) or "dms_v1"
data_root = pack_task_data_root(root, pack, tcfg["task_dir"])
sk = split_kwargs(reg, args)
@@ -449,23 +453,42 @@ def ingest_all_sources(root: Path, reg: dict, task: str, args: argparse.Namespac
def ingest_all_inbox(root: Path, reg: dict, args: argparse.Namespace) -> None:
from task_registry import inbox_dir
pack = getattr(args, "pack", None) or "dms_v1"
for task, tcfg in reg["tasks"].items():
inbox = root / tcfg.get("inbox", f"inbox/{task}")
if not inbox.is_dir():
continue
batches = sorted(d for d in inbox.iterdir() if d.is_dir())
if not batches:
continue
print(f"\n>>> inbox {task}: {len(batches)} batch(es)")
for batch in batches:
ingest_one(root, reg, task, batch, args)
if not args.dry_run:
append_log(root, {"src": str(batch), "task": task, "pack": pack, "via": "inbox"})
if tcfg.get("type") == "multi":
for mode in (tcfg.get("modes") or {}):
inbox = inbox_dir(root, task, mode, reg)
if not inbox.is_dir():
continue
batches = sorted(d for d in inbox.iterdir() if d.is_dir())
if not batches:
continue
print(f"\n>>> inbox {task}/{mode}: {len(batches)} batch(es)")
args.submode = mode
for batch in batches:
ingest_one(root, reg, task, batch, args)
if not args.dry_run:
append_log(root, {"src": str(batch), "task": task, "mode": mode, "pack": pack, "via": "inbox"})
else:
inbox = inbox_dir(root, task, None, reg)
if not inbox.is_dir():
continue
batches = sorted(d for d in inbox.iterdir() if d.is_dir())
if not batches:
continue
print(f"\n>>> inbox {task}: {len(batches)} batch(es)")
for batch in batches:
ingest_one(root, reg, task, batch, args)
if not args.dry_run:
append_log(root, {"src": str(batch), "task": task, "pack": pack, "via": "inbox"})
def main() -> None:
p = argparse.ArgumentParser(description="DMS 全任务增量接入")
p.add_argument("--task", help="registry 任务名；与 --all-inbox 二选一")
p.add_argument("--task", help="registry 任务名（forward 等 multi 任务需配合 --submode）")
p.add_argument("--submode", choices=("detect", "classify"), help="multi 任务子模式，如 forward 的 detect/classify")
p.add_argument("--src", type=Path, help="新数据目录")
p.add_argument("--all-inbox", action="store_true", help="处理所有 inbox/<task>/* 批次")
p.add_argument("--all-sources", action="store_true", help="处理任务 data/sources/* 下所有待合并批次")

77
datasets/dms/scripts/migrate_dam_layout.py Normal file
View File

@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""将 dam / dam_0417 合并为 dam/batch_0516、dam/batch_0417（默认符号链接）。"""
from __future__ import annotations
import argparse
import shutil
from pathlib import Path
def link_or_move(src: Path, dst: Path, *, move: bool) -> None:
if not src.is_dir():
print(f" skip（不存在）: {src}")
return
if dst.exists():
print(f" 已存在: {dst}")
return
dst.parent.mkdir(parents=True, exist_ok=True)
if move:
shutil.move(str(src), str(dst))
print(f" moved {src} -> {dst}")
else:
dst.symlink_to(src.resolve())
print(f" symlink {dst} -> {src.resolve()}")
def migrate_pack(pack_dir: Path, *, move: bool) -> None:
dam_root = pack_dir / "dam"
dam_0417 = pack_dir / "dam_0417"
batch_0516 = dam_root / "batch_0516"
batch_0417 = dam_root / "batch_0417"
if batch_0516.exists() and batch_0417.exists():
print(" dam 已迁移，跳过")
return
# 当前 dam 为扁平 YOLO 布局（images/labels 在根下）
if dam_root.is_dir() and (dam_root / "images").is_dir() and not batch_0516.exists():
stash = pack_dir / "_dam_stash_0516"
if stash.exists():
print(f" 清理旧 stash: {stash}")
if stash.is_symlink():
stash.unlink()
else:
shutil.rmtree(stash)
shutil.move(str(dam_root), str(stash))
dam_root.mkdir(parents=True)
link_or_move(stash, batch_0516, move=move)
if dam_0417.is_dir():
link_or_move(dam_0417, batch_0417, move=move)
def migrate_inbox(dms_root: Path, *, move: bool) -> None:
for old, new in (
("dam", "dam/batch_0516"),
("dam_0417", "dam/batch_0417"),
):
src = dms_root / "inbox" / old
dst = dms_root / "inbox" / new
if src.is_dir():
link_or_move(src, dst, move=move)
def main() -> None:
p = argparse.ArgumentParser()
p.add_argument("--pack-dir", type=Path, required=True)
p.add_argument("--dms-root", type=Path, help="datasets/dms 根，迁移 inbox")
p.add_argument("--move", action="store_true")
args = p.parse_args()
migrate_pack(args.pack_dir.resolve(), move=args.move)
if args.dms_root:
migrate_inbox(args.dms_root.resolve(), move=args.move)
print("完成。请运行 refresh_yaml.py --task dam 并刷新 catalog。")
if __name__ == "__main__":
main()

57
datasets/dms/scripts/migrate_forward_layout.py Normal file
View File

@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""将 isa / isa_class 目录迁入 forward/detect、forward/classify（默认符号链接，保留原数据）。"""
from __future__ import annotations
import argparse
import shutil
from pathlib import Path
def link_or_move(src: Path, dst: Path, *, move: bool) -> None:
if not src.is_dir():
print(f" skip（不存在）: {src}")
return
if dst.exists():
print(f" 已存在: {dst}")
return
dst.parent.mkdir(parents=True, exist_ok=True)
if move:
shutil.move(str(src), str(dst))
print(f" moved {src} -> {dst}")
else:
dst.symlink_to(src.resolve())
print(f" symlink {dst} -> {src.resolve()}")
def migrate_pack(pack_dir: Path, *, move: bool) -> None:
forward = pack_dir / "forward"
forward.mkdir(parents=True, exist_ok=True)
link_or_move(pack_dir / "isa", forward / "detect", move=move)
link_or_move(pack_dir / "isa_class", forward / "classify", move=move)
def migrate_inbox(dms_root: Path, *, move: bool) -> None:
for old, new in (
("isa", "forward/detect"),
("isa_class", "forward/classify"),
):
src = dms_root / "inbox" / old
dst = dms_root / "inbox" / new
if src.is_dir():
link_or_move(src, dst, move=move)
def main() -> None:
p = argparse.ArgumentParser()
p.add_argument("--pack-dir", type=Path, required=True, help="如 .../packs/dms_v1")
p.add_argument("--dms-root", type=Path, help="datasets/dms 根，迁移 inbox")
p.add_argument("--move", action="store_true", help="移动而非符号链接")
args = p.parse_args()
migrate_pack(args.pack_dir.resolve(), move=args.move)
if args.dms_root:
migrate_inbox(args.dms_root.resolve(), move=args.move)
print("完成。请运行 refresh_yaml.py 并刷新平台 catalog。")
if __name__ == "__main__":
main()

67
datasets/dms/scripts/refresh_yaml.py
View File

@@ -1,6 +1,5 @@
#!/usr/bin/env python3
"""按 workflow active_packs 生成 manifests/yaml_active/*.yaml（可多包合并 train/val）。"""
from __future__ import annotations
import argparse
@@ -11,10 +10,8 @@ import yaml
SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))
from pack_registry import ( # noqa: E402
load_active_pack_names,
resolve_pack_dir,
)
from pack_registry import load_active_pack_names, resolve_pack_dir # noqa: E402
from task_registry import get_mode_config, load_registry, train_yaml_key # noqa: E402
def fmt_names(names) -> str:
@@ -39,13 +36,13 @@ def pack_task_root(root: Path, pack_name: str, task_dir: str) -> Path:
def build_detect_pose_yaml(
task: str,
tcfg: dict,
yaml_key: str,
mcfg: dict,
root: Path,
pack_names: list[str],
typ: str,
) -> str:
task_dir = tcfg["task_dir"]
task_dir = mcfg["task_dir"]
bases = []
train_paths = []
val_paths = []
@@ -59,28 +56,28 @@ def build_detect_pose_yaml(
val_paths.append(str((base / "images" / "val").resolve()))
if not bases:
raise SystemExit(f"{task}: 无可用数据包目录")
raise SystemExit(f"{yaml_key}: 无可用数据包目录")
lines = [
f"# {task} — packs: {', '.join(pack_names)}",
f"# {yaml_key} — packs: {', '.join(pack_names)}",
f"path: {bases[0]}",
yaml_list("train", train_paths),
yaml_list("val", val_paths),
"",
]
if typ == "pose":
lines.insert(4, f"kpt_shape: {tcfg.get('kpt_shape', [37, 3])}")
lines.insert(4, f"kpt_shape: {mcfg.get('kpt_shape', [37, 3])}")
else:
lines.extend([f"nc: {tcfg['nc']}", fmt_names(tcfg["names"]), ""])
lines.extend([f"nc: {mcfg['nc']}", fmt_names(mcfg["names"]), ""])
return "\n".join(lines)
def build_classify_yaml(task: str, tcfg: dict, root: Path, pack_names: list[str]) -> str:
task_dir = tcfg["task_dir"]
def build_classify_yaml(yaml_key: str, mcfg: dict, root: Path, pack_names: list[str]) -> str:
task_dir = mcfg["task_dir"]
if len(pack_names) > 1:
print(f" warn {task}: classify 暂用首个包 {pack_names[0]}（多包请先合并目录）")
print(f" warn {yaml_key}: classify 暂用首个包 {pack_names[0]}（多包请先合并目录）")
base = pack_task_root(root, pack_names[0], task_dir)
return f"""# {task} — pack: {pack_names[0]}
return f"""# {yaml_key} — pack: {pack_names[0]}
path: {base.resolve()}
train: train
val: val
@@ -88,39 +85,49 @@ test: test
"""
def iter_yaml_jobs(reg: dict, only_task: str | None = None):
tasks = load_registry(reg)
if only_task:
if only_task not in tasks:
raise SystemExit(f"未知 task: {only_task}")
tasks = {only_task: tasks[only_task]}
for task, tcfg in tasks.items():
if tcfg.get("type") == "multi":
for mode in (tcfg.get("modes") or {}):
mcfg = get_mode_config(task, mode, reg)
key = train_yaml_key(task, mode, reg)
yield key, mcfg
else:
yield task, tcfg
def main() -> None:
p = argparse.ArgumentParser()
p.add_argument("--root", type=Path, default=SCRIPT_DIR.parent)
p.add_argument("--packs", help="逗号分隔，覆盖 workflow active_packs")
p.add_argument("--task", help="只生成某一任务")
p.add_argument("--task", help="只生成某一任务（multi 会生成全部 mode）")
args = p.parse_args()
root = args.root.resolve()
reg = yaml.safe_load((root / "datasets.registry.yaml").read_text(encoding="utf-8"))
cli = [x.strip() for x in args.packs.split(",")] if args.packs else None
pack_names = load_active_pack_names(root, cli)
if not pack_names:
raise SystemExit("active_packs 为空，请编辑 ML/workflow.registry.yaml 或 --packs")
raise SystemExit("active_packs 为空，请编辑 workflow.registry.yaml 或 --packs")
out_dir = root / "manifests" / "yaml_active"
out_dir.mkdir(parents=True, exist_ok=True)
print(f"active_packs: {pack_names}")
tasks = reg["tasks"]
if args.task:
if args.task not in tasks:
raise SystemExit(f"未知 task: {args.task}")
tasks = {args.task: tasks[args.task]}
for task, tcfg in tasks.items():
typ = tcfg["type"]
for yaml_key, mcfg in iter_yaml_jobs(reg, args.task):
typ = mcfg["type"]
if typ in ("detect", "pose"):
content = build_detect_pose_yaml(task, tcfg, root, pack_names, typ)
content = build_detect_pose_yaml(yaml_key, mcfg, root, pack_names, typ)
elif typ == "classify":
content = build_classify_yaml(task, tcfg, root, pack_names)
content = build_classify_yaml(yaml_key, mcfg, root, pack_names)
else:
print(f" skip {task}: type {typ}")
print(f" skip {yaml_key}: type {typ}")
continue
out = out_dir / f"{task}.yaml"
out = out_dir / f"{yaml_key}.yaml"
out.write_text(content, encoding="utf-8")
print(f" wrote {out.relative_to(root)}")

166
datasets/dms/scripts/task_registry.py Normal file
View File

@@ -0,0 +1,166 @@
"""DMS 任务注册表：domain 分组、multi 任务（前向 detect+classify）、旧 ID 别名。"""
from __future__ import annotations
from typing import Any
DOMAIN_LABELS = {
"dms": "舱内 DMS",
"forward": "前向 ADAS",
}
# 报表 / 旧目录名 -> (task, mode)
REPORT_TASK_ALIASES: dict[str, tuple[str, str | None]] = {
"isa": ("forward", "detect"),
"isa_detect": ("forward", "detect"),
"isa_class": ("forward", "classify"),
"isa_class_0116": ("forward", "classify"),
"dam_0417": ("dam", "batch_0417"),
}
LEGACY_TASK_ALIASES: dict[str, tuple[str, str | None]] = {
"isa": ("forward", "detect"),
"isa_class": ("forward", "classify"),
"dam_0417": ("dam", "batch_0417"),
}
def load_registry(reg: dict) -> dict[str, Any]:
return reg.get("tasks") or {}
def resolve_task_id(task: str, mode: str | None = None) -> tuple[str, str | None]:
"""用户/历史 task ID -> (canonical_task, mode)。"""
if task in LEGACY_TASK_ALIASES:
t, m = LEGACY_TASK_ALIASES[task]
return t, mode or m
return task, mode
def report_task_key(task: str, mode: str | None = None) -> str:
"""catalog 报表 CSV 中的任务列名。"""
t, m = resolve_task_id(task, mode)
if t == "forward" and m == "detect":
return "isa"
if t == "forward" and m == "classify":
return "isa_class"
if t == "dam" and m == "batch_0516":
return "dam"
if t == "dam" and m == "batch_0417":
return "dam_0417"
return task
def train_yaml_key(task: str, mode: str | None, reg: dict) -> str:
"""manifests/yaml_active 与 train.sh 使用的文件名（不含 .yaml）。"""
task, mode = resolve_task_id(task, mode)
tcfg = load_registry(reg)[task]
if tcfg.get("type") == "multi":
if not mode:
raise ValueError(f"任务 {task} 需指定 mode（detect / classify）")
return f"{task}__{mode}"
return task
def get_mode_config(task: str, mode: str | None, reg: dict) -> dict[str, Any]:
task, mode = resolve_task_id(task, mode)
tcfg = load_registry(reg)[task]
if tcfg.get("type") != "multi":
return {**tcfg, "task": task, "mode": None}
modes = tcfg.get("modes") or {}
if not mode:
raise ValueError(f"任务 {task} 需指定 mode")
if mode not in modes:
raise ValueError(f"未知 mode: {task}/{mode}")
mcfg = dict(modes[mode])
mcfg["task"] = task
mcfg["mode"] = mode
mcfg["task_dir"] = f"{tcfg.get('task_dir', task)}/{mcfg.get('subdir', mode)}"
mcfg["domain"] = tcfg.get("domain")
mcfg["label"] = mcfg.get("label") or tcfg.get("label")
return mcfg
def task_data_dir(pack_dir, task: str, mode: str | None, reg: dict):
from pathlib import Path
mcfg = get_mode_config(task, mode, reg)
return Path(pack_dir) / mcfg["task_dir"]
def inbox_dir(root, task: str, mode: str | None, reg: dict):
from pathlib import Path
task, mode = resolve_task_id(task, mode)
tcfg = load_registry(reg)[task]
if tcfg.get("type") == "multi":
mcfg = tcfg["modes"][mode or ""]
rel = mcfg.get("inbox") or f"inbox/{task}/{mode}"
return Path(root) / rel
return Path(root) / (tcfg.get("inbox") or f"inbox/{task}")
def iter_catalog_tasks(reg: dict) -> list[tuple[str, dict[str, Any]]]:
"""catalog 顶层任务列表。"""
out: list[tuple[str, dict[str, Any]]] = []
for task, tcfg in load_registry(reg).items():
entry = {
"domain": tcfg.get("domain", "dms"),
"domain_label": DOMAIN_LABELS.get(tcfg.get("domain", "dms"), tcfg.get("domain", "dms")),
"label": tcfg.get("label", task),
"type": tcfg.get("type"),
}
if tcfg.get("type") == "multi":
entry["modes"] = {}
for mode, mcfg in (tcfg.get("modes") or {}).items():
entry["modes"][mode] = {
"label": mcfg.get("label", mode),
"type": mcfg.get("type"),
"nc": mcfg.get("nc"),
"names": mcfg.get("names"),
"packs": [],
"class_counts": {},
}
else:
entry["nc"] = tcfg.get("nc")
entry["names"] = tcfg.get("names")
entry["packs"] = []
entry["class_counts"] = {}
out.append((task, entry))
return out
def map_report_task(report_name: str) -> tuple[str, str | None]:
if report_name in REPORT_TASK_ALIASES:
return REPORT_TASK_ALIASES[report_name]
return report_name, None
def task_defs_for_pending(reg: dict) -> dict[str, Any]:
"""平台 pending API 的 task_defs。"""
defs: dict[str, Any] = {}
for task, tcfg in load_registry(reg).items():
if tcfg.get("type") == "multi":
defs[task] = {
"type": "multi",
"domain": tcfg.get("domain", "dms"),
"label": tcfg.get("label", task),
"modes": {
m: {
"type": mc.get("type"),
"nc": mc.get("nc"),
"names": mc.get("names"),
"task_dir": f"{tcfg.get('task_dir', task)}/{mc.get('subdir', m)}",
}
for m, mc in (tcfg.get("modes") or {}).items()
},
}
else:
defs[task] = {
"type": tcfg.get("type"),
"domain": tcfg.get("domain", "dms"),
"label": tcfg.get("label", task),
"nc": tcfg.get("nc"),
"names": tcfg.get("names"),
"task_dir": tcfg.get("task_dir", task),
}
return defs

143
datasets/dms/scripts/test_export_ls_to_yolo.py Normal file
View File

@@ -0,0 +1,143 @@
#!/usr/bin/env python3
"""export_ls_to_yolo 单元测试（无 pytest 依赖）。"""
from __future__ import annotations
import hashlib
import json
import sys
import tempfile
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))
from export_ls_to_yolo import ( # noqa: E402
convert_regions_to_yolo_lines,
export_batch,
)
from ingest_incremental import validate_detect_label, validate_pose_label # noqa: E402
def _task_id(rel: str) -> str:
return hashlib.sha256(rel.encode()).hexdigest()[:16]
def test_detect_conversion() -> None:
regions = [
{
"type": "rectanglelabels",
"value": {
"x": 10.0,
"y": 20.0,
"width": 30.0,
"height": 40.0,
"rectanglelabels": ["face"],
},
}
]
lines = convert_regions_to_yolo_lines(
regions,
mode="detect",
class_map={"face": 0, "eye_open": 1},
)
assert len(lines) == 1
parts = lines[0].split()
assert len(parts) == 5
assert parts[0] == "0"
assert abs(float(parts[1]) - 0.25) < 1e-5 # cx = (10+15)/100
assert abs(float(parts[2]) - 0.40) < 1e-5 # cy = (20+20)/100
err = validate_detect_label("\n".join(lines), 4)
assert err is None, err
def test_pose_conversion() -> None:
regions = [
{
"type": "rectanglelabels",
"value": {
"x": 10.0,
"y": 20.0,
"width": 30.0,
"height": 40.0,
"rectanglelabels": ["face"],
},
},
{
"type": "keypointlabels",
"value": {"x": 35.6, "y": 52.9, "width": 0.5, "keypointlabels": ["kp_01"]},
},
{
"type": "keypointlabels",
"value": {"x": 50.0, "y": 50.0, "width": 0.5, "keypointlabels": ["kp_10"]},
},
]
kpt_map = {f"kp_{i:02d}": i for i in range(37)}
lines = convert_regions_to_yolo_lines(
regions,
mode="pose",
class_map={"face": 0},
kpt_map=kpt_map,
kpt_shape=[37, 3],
)
assert len(lines) == 1
parts = lines[0].split()
assert len(parts) == 116
assert parts[0] == "0"
# kp_01 at index 1 -> fields 5+3..5+5
assert abs(float(parts[8]) - 0.356) < 1e-3
assert abs(float(parts[9]) - 0.529) < 1e-3
assert parts[10] == "2.000000"
err = validate_pose_label("\n".join(lines), [37, 3])
assert err is None, err
def test_export_batch_end_to_end() -> None:
with tempfile.TemporaryDirectory() as tmp:
batch = Path(tmp)
img_rel = "images/train/sample.jpg"
img_path = batch / img_rel
img_path.parent.mkdir(parents=True)
img_path.write_bytes(b"\xff\xd8\xff")
tid = _task_id(img_rel)
ann = {
"task_id": tid,
"result": [
{
"type": "rectanglelabels",
"value": {
"x": 10.0,
"y": 20.0,
"width": 30.0,
"height": 40.0,
"rectanglelabels": ["face"],
},
},
{
"type": "keypointlabels",
"value": {"x": 25.0, "y": 40.0, "width": 0.5, "keypointlabels": ["kp_00"]},
},
],
}
ann_dir = batch / "labels" / "ls_annotations"
ann_dir.mkdir(parents=True)
(ann_dir / f"{tid}.json").write_text(json.dumps(ann), encoding="utf-8")
result = export_batch(batch, "addw_face", mode="pose")
assert result["written"] == 1
out = batch / "labels" / "train" / "sample.txt"
assert out.is_file()
parts = out.read_text().strip().split()
assert len(parts) == 116
def main() -> int:
test_detect_conversion()
test_pose_conversion()
test_export_batch_end_to_end()
print("OK export_ls_to_yolo tests")
return 0
if __name__ == "__main__":
raise SystemExit(main())

35
datasets/dms/scripts/train.sh
View File

@@ -17,20 +17,21 @@ TASK="${1:?用法: $0 <task> [full|continue]}"
TRAIN_MODE="${2:-full}"
REG="$DATASET_ROOT/datasets.registry.yaml"
YAML="$DATASET_ROOT/manifests/yaml_active/${TASK}.yaml"
VERSIONS="$DATASET_ROOT/manifests/train_versions.yaml"
SUBMODE="${SUBMODE:-}"
if [[ ! -f "$YAML" ]]; then
echo "找不到 yaml: $YAML"
exit 1
fi
read -r TYPE MODE MODEL EPOCHS LR0 IMGSZ RUN_SUFFIX <<< "$(python3 - <<PY
import yaml
read -r YAML_KEY TYPE MODE MODEL EPOCHS LR0 IMGSZ RUN_SUFFIX <<< "$(python3 - <<PY
import sys
from pathlib import Path
import yaml
sys.path.insert(0, str(Path("$DATASET_ROOT/scripts")))
from task_registry import get_mode_config, resolve_task_id, train_yaml_key
reg = yaml.safe_load(Path("$REG").read_text())
tcfg = reg["tasks"]["$TASK"]
typ = tcfg["type"]
task, sub = resolve_task_id("$TASK", "$SUBMODE" or None)
mcfg = get_mode_config(task, sub, reg)
typ = mcfg["type"]
yaml_key = train_yaml_key(task, sub, reg)
print(yaml_key, end=" ")
train_mode = "$TRAIN_MODE" if "$TRAIN_MODE" in ("full", "continue") else reg.get("train", {}).get("mode", "full")
t = reg.get("train", {}).get(typ, reg.get("train_defaults", {}).get(typ, {}))
if train_mode == "continue":
@@ -49,6 +50,12 @@ print(typ, mode, model, epochs, lr0, imgsz, suffix)
PY
)"
YAML="$DATASET_ROOT/manifests/yaml_active/${YAML_KEY}.yaml"
if [[ ! -f "$YAML" ]]; then
echo "找不到 yaml: $YAML（请先 refresh_yaml.py）"
exit 1
fi
# continue 模式：warm_start 为空则读 train_versions.yaml
if [[ "$TRAIN_MODE" == "continue" && ( "$MODEL" == "null" || "$MODEL" == "None" || -z "$MODEL" ) ]]; then
MODEL=$(python3 - <<PY 2>/dev/null || true
@@ -57,7 +64,7 @@ from pathlib import Path
p = Path("$VERSIONS")
if p.is_file():
v = yaml.safe_load(p.read_text()) or {}
c = v.get("$TASK", {}).get("current")
c = v.get("$YAML_KEY", {}).get("current")
if c: print(c)
PY
)
@@ -68,9 +75,9 @@ if [[ "$TRAIN_MODE" == "continue" && ( -z "$MODEL" || "$MODEL" == "null" ) ]]; t
exit 1
fi
RUN_NAME="${TASK}_${RUN_SUFFIX}_$(date +%Y%m%d)"
RUN_NAME="${YAML_KEY}_${RUN_SUFFIX}_$(date +%Y%m%d)"
echo "task=$TASK type=$TYPE yolo_mode=$MODE train_mode=$TRAIN_MODE"
echo "task=$TASK submode=$SUBMODE yaml_key=$YAML_KEY type=$TYPE yolo_mode=$MODE train_mode=$TRAIN_MODE"
echo "data=$YAML"
echo "model=$MODEL epochs=$EPOCHS lr0=$LR0 imgsz=$IMGSZ name=$RUN_NAME"
@@ -93,4 +100,4 @@ yolo "$MODE" train \
BEST="runs/${MODE}/${RUN_NAME}/weights/best.pt"
echo "完成: $BEST"
echo "请更新 manifests/train_versions.yaml 中 $TASK.current = $BEST"
echo "请更新 manifests/train_versions.yaml 中 $YAML_KEY.current = $BEST"

45
datasets/dms/versions/v1.json Normal file
View File

@@ -0,0 +1,45 @@
{
"version_id": "v1",
"project": "dms",
"created_at": "2026-06-01T09:51:41.352091+00:00",
"description": "初始版本",
"author": "test",
"parent_version": null,
"summary": {
"packs_count": 2,
"total_images": 12814,
"total_labels": 6000,
"batches_count": 4
},
"packs": {
"dms_v1": {
"train_images": 11464,
"val_images": 1350,
"test_images": 0,
"class_counts": {},
"tasks": [
"ddaw",
"addw",
"addw_face"
]
},
"dms_v2": {
"train_images": 0,
"val_images": 0,
"test_images": 0,
"class_counts": {},
"tasks": [
"ddaw",
"addw",
"addw_face"
]
}
},
"batches": [
"20260525_pilot",
"addw_face",
"ddaw",
"labels"
],
"diff": null
}

50
datasets/dms/versions/v2.json Normal file
View File

@@ -0,0 +1,50 @@
{
"version_id": "v2",
"project": "dms",
"created_at": "2026-06-01T09:51:51.218991+00:00",
"description": "第二个版本",
"author": "test",
"parent_version": "v1",
"summary": {
"packs_count": 2,
"total_images": 12814,
"total_labels": 6000,
"batches_count": 4
},
"packs": {
"dms_v1": {
"train_images": 11464,
"val_images": 1350,
"test_images": 0,
"class_counts": {},
"tasks": [
"ddaw",
"addw",
"addw_face"
]
},
"dms_v2": {
"train_images": 0,
"val_images": 0,
"test_images": 0,
"class_counts": {},
"tasks": [
"ddaw",
"addw",
"addw_face"
]
}
},
"batches": [
"20260525_pilot",
"addw_face",
"ddaw",
"labels"
],
"diff": {
"added_packs": [],
"removed_packs": [],
"added_batches": [],
"removed_batches": []
}
}

42
datasets/labeling.registry.yaml Normal file
View File

@@ -0,0 +1,42 @@
version: 1
profiles:
ddaw:
scope_key: "dms:ddaw"
editor_template: dms_detect.xml
export_default: yolo
ml_adapter: dms_yolo
addw:
scope_key: "dms:addw"
editor_template: dms_detect.xml
export_default: yolo
ml_adapter: dms_yolo
addw_face:
scope_key: "dms:addw_face"
editor_template: dms_pose.xml
export_default: yolo_pose
ml_adapter: dms_yolo_pose
dam__batch_0516:
scope_key: "dms:dam:batch_0516"
editor_template: dam_15cls.xml
export_default: yolo
ml_adapter: dms_yolo
dam__batch_0417:
scope_key: "dms:dam:batch_0417"
editor_template: dam_15cls.xml
export_default: yolo
ml_adapter: dms_yolo
forward__detect:
scope_key: "dms:forward:detect"
editor_template: forward_4cls.xml
export_default: yolo
ml_adapter: dms_yolo
forward__classify:
scope_key: "dms:forward:classify"
editor_template: forward_cls.xml
export_default: folder_cls
ml_adapter: dms_yolo_cls
lane__lane_v1:
scope_key: "lane:lane_v1"
editor_template: lane_ufld_mask.xml
export_default: lane_gt_txt
ml_adapter: lane_ufld

1
datasets/lane Symbolic link
View File

@@ -0,0 +1 @@
../../workspace/lane

0
datasets/lane/DATASETS_LAYOUT.md → datasets/lane.embedded.bak/DATASETS_LAYOUT.md
View File

0
datasets/lane/README.md → datasets/lane.embedded.bak/README.md
View File

0
datasets/lane/datasets_registry.json → datasets/lane.embedded.bak/datasets_registry.json
View File

0
datasets/lane/lists_merged/train__DATASET.txt → datasets/lane.embedded.bak/lists_merged/train__DATASET.txt
View File

0
datasets/lane/scripts/build_ufld_dataset.py → datasets/lane.embedded.bak/scripts/build_ufld_dataset.py
View File

0
datasets/lane/scripts/build_ufld_pack.py → datasets/lane.embedded.bak/scripts/build_ufld_pack.py
View File

0
datasets/lane/scripts/merge_ufld_lists.py → datasets/lane.embedded.bak/scripts/merge_ufld_lists.py
View File

0
datasets/lane/scripts/rename_ufld_dataset.py → datasets/lane.embedded.bak/scripts/rename_ufld_dataset.py
View File