feat: HSAP platform v2 — modular navigation, quality review, audit log, world model simulation
Major changes: - New frontend (platform/web/): Vite + React 18 + TypeScript + Tailwind - 4-module navigation: 数据送标 / 模型管理 / 车队管理 / 系统管理 - Data catalog with charts (DMS/ADAS/Lane 3-tab view) - Quality review workflow (标注质检): Good/Fine/Bad scoring with auto-advance - Audit enhancements: batch operations, rejection categories, Feishu notifications - Operation audit log (操作日志) - World model simulation studio (仿真工坊) - Dataset version management with snapshots and diff - ADAS 7-class dataset integration (138K images organized + compressed) - User management with Feishu integration and pagination - CRUD/search/filter on all pages, card layout redesign - PIL-optimized image overlay rendering - Auto-snapshot on build, in_review workflow stage - Removed embedded algorithm code (now in workspace)
This commit is contained in:
43
datasets/dms/configs/keypoint_order/addw_face_37.yaml
Normal file
43
datasets/dms/configs/keypoint_order/addw_face_37.yaml
Normal file
@@ -0,0 +1,43 @@
|
||||
# addw_face：37 点顺序与 Label Studio dms_pose.xml 中 kp_XX 标签一一对应
|
||||
task: addw_face
|
||||
kpt_shape: [37, 3]
|
||||
class_names:
|
||||
0: face
|
||||
keypoints:
|
||||
- {id: 0, label: kp_00}
|
||||
- {id: 1, label: kp_01}
|
||||
- {id: 2, label: kp_02}
|
||||
- {id: 3, label: kp_03}
|
||||
- {id: 4, label: kp_04}
|
||||
- {id: 5, label: kp_05}
|
||||
- {id: 6, label: kp_06}
|
||||
- {id: 7, label: kp_07}
|
||||
- {id: 8, label: kp_08}
|
||||
- {id: 9, label: kp_09}
|
||||
- {id: 10, label: kp_10}
|
||||
- {id: 11, label: kp_11}
|
||||
- {id: 12, label: kp_12}
|
||||
- {id: 13, label: kp_13}
|
||||
- {id: 14, label: kp_14}
|
||||
- {id: 15, label: kp_15}
|
||||
- {id: 16, label: kp_16}
|
||||
- {id: 17, label: kp_17}
|
||||
- {id: 18, label: kp_18}
|
||||
- {id: 19, label: kp_19}
|
||||
- {id: 20, label: kp_20}
|
||||
- {id: 21, label: kp_21}
|
||||
- {id: 22, label: kp_22}
|
||||
- {id: 23, label: kp_23}
|
||||
- {id: 24, label: kp_24}
|
||||
- {id: 25, label: kp_25}
|
||||
- {id: 26, label: kp_26}
|
||||
- {id: 27, label: kp_27}
|
||||
- {id: 28, label: kp_28}
|
||||
- {id: 29, label: kp_29}
|
||||
- {id: 30, label: kp_30}
|
||||
- {id: 31, label: kp_31}
|
||||
- {id: 32, label: kp_32}
|
||||
- {id: 33, label: kp_33}
|
||||
- {id: 34, label: kp_34}
|
||||
- {id: 35, label: kp_35}
|
||||
- {id: 36, label: kp_36}
|
||||
7
datasets/dms/configs/label_studio/dam_15cls.xml
Normal file
7
datasets/dms/configs/label_studio/dam_15cls.xml
Normal file
@@ -0,0 +1,7 @@
|
||||
<View>
|
||||
<Image name="image" value="$image"/>
|
||||
<RectangleLabels name="label" toName="image">
|
||||
<Label value="face" category="0"/>
|
||||
<Label value="driver" category="14"/>
|
||||
</RectangleLabels>
|
||||
</View>
|
||||
9
datasets/dms/configs/label_studio/dms_detect.xml
Normal file
9
datasets/dms/configs/label_studio/dms_detect.xml
Normal file
@@ -0,0 +1,9 @@
|
||||
<View>
|
||||
<Image name="image" value="$image"/>
|
||||
<RectangleLabels name="label" toName="image">
|
||||
<Label value="face"/>
|
||||
<Label value="eye_open"/>
|
||||
<Label value="nod_eye"/>
|
||||
<Label value="nod_face"/>
|
||||
</RectangleLabels>
|
||||
</View>
|
||||
46
datasets/dms/configs/label_studio/dms_pose.xml
Normal file
46
datasets/dms/configs/label_studio/dms_pose.xml
Normal file
@@ -0,0 +1,46 @@
|
||||
<View>
|
||||
<Image name="image" value="$image" zoom="true"/>
|
||||
<RectangleLabels name="bbox" toName="image">
|
||||
<Label value="face"/>
|
||||
</RectangleLabels>
|
||||
<KeyPointLabels name="kpts" toName="image" choice="single">
|
||||
<Label value="kp_00" maxUsages="1"/>
|
||||
<Label value="kp_01" maxUsages="1"/>
|
||||
<Label value="kp_02" maxUsages="1"/>
|
||||
<Label value="kp_03" maxUsages="1"/>
|
||||
<Label value="kp_04" maxUsages="1"/>
|
||||
<Label value="kp_05" maxUsages="1"/>
|
||||
<Label value="kp_06" maxUsages="1"/>
|
||||
<Label value="kp_07" maxUsages="1"/>
|
||||
<Label value="kp_08" maxUsages="1"/>
|
||||
<Label value="kp_09" maxUsages="1"/>
|
||||
<Label value="kp_10" maxUsages="1"/>
|
||||
<Label value="kp_11" maxUsages="1"/>
|
||||
<Label value="kp_12" maxUsages="1"/>
|
||||
<Label value="kp_13" maxUsages="1"/>
|
||||
<Label value="kp_14" maxUsages="1"/>
|
||||
<Label value="kp_15" maxUsages="1"/>
|
||||
<Label value="kp_16" maxUsages="1"/>
|
||||
<Label value="kp_17" maxUsages="1"/>
|
||||
<Label value="kp_18" maxUsages="1"/>
|
||||
<Label value="kp_19" maxUsages="1"/>
|
||||
<Label value="kp_20" maxUsages="1"/>
|
||||
<Label value="kp_21" maxUsages="1"/>
|
||||
<Label value="kp_22" maxUsages="1"/>
|
||||
<Label value="kp_23" maxUsages="1"/>
|
||||
<Label value="kp_24" maxUsages="1"/>
|
||||
<Label value="kp_25" maxUsages="1"/>
|
||||
<Label value="kp_26" maxUsages="1"/>
|
||||
<Label value="kp_27" maxUsages="1"/>
|
||||
<Label value="kp_28" maxUsages="1"/>
|
||||
<Label value="kp_29" maxUsages="1"/>
|
||||
<Label value="kp_30" maxUsages="1"/>
|
||||
<Label value="kp_31" maxUsages="1"/>
|
||||
<Label value="kp_32" maxUsages="1"/>
|
||||
<Label value="kp_33" maxUsages="1"/>
|
||||
<Label value="kp_34" maxUsages="1"/>
|
||||
<Label value="kp_35" maxUsages="1"/>
|
||||
<Label value="kp_36" maxUsages="1"/>
|
||||
</KeyPointLabels>
|
||||
</View>
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
# DMS 数据包登记（训练用哪些包由 ML/workflow.registry.yaml 的 active_packs 决定）
|
||||
|
||||
schema: dms-multi-pack-v1
|
||||
|
||||
packs:
|
||||
- name: dms_v1
|
||||
path: packs/dms_v1
|
||||
role: baseline
|
||||
frozen: true
|
||||
|
||||
# 新建增量包示例（add/build 写入，需 enable 后才参与训练）:
|
||||
# - name: dms_v2
|
||||
# path: packs/dms_v2
|
||||
|
||||
- name: dms_v1
|
||||
path: packs/dms_v1
|
||||
role: baseline
|
||||
frozen: true
|
||||
- name: dms_v2
|
||||
path: packs/dms_v2
|
||||
role: incremental
|
||||
- name: adas_v1
|
||||
path: packs/adas_v1
|
||||
role: adas_forward
|
||||
note: 前向 ADAS 7 类检测数据集 (53GB)
|
||||
aliases:
|
||||
DMS_v1_AddBy_Luchengfang_20260520: dms_v1
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# 任务定义 + 训练超参。数据包路径见 data_packs.yaml；训练用哪些包见 ML/workflow.registry.yaml active_packs
|
||||
# 任务定义 + 训练超参。数据包路径见 data_packs.yaml；训练用哪些包见 workflow.registry.yaml active_packs
|
||||
# domain: dms=舱内驾驶员监控 | forward=前向交通标志（检测+细分类合一任务）
|
||||
|
||||
version: 2
|
||||
version: 3
|
||||
dataset_root: .
|
||||
|
||||
split:
|
||||
@@ -47,6 +48,8 @@ train:
|
||||
|
||||
tasks:
|
||||
ddaw:
|
||||
domain: dms
|
||||
label: DDAW 疲劳检测
|
||||
type: detect
|
||||
task_dir: ddaw
|
||||
nc: 9
|
||||
@@ -55,6 +58,8 @@ tasks:
|
||||
extra_train: []
|
||||
|
||||
addw:
|
||||
domain: dms
|
||||
label: ADDW 分心检测
|
||||
type: detect
|
||||
task_dir: addw
|
||||
nc: 4
|
||||
@@ -63,6 +68,8 @@ tasks:
|
||||
extra_train: []
|
||||
|
||||
addw_face:
|
||||
domain: dms
|
||||
label: ADDW 人脸关键点
|
||||
type: pose
|
||||
task_dir: addw_face
|
||||
kpt_shape: [37, 3]
|
||||
@@ -70,32 +77,56 @@ tasks:
|
||||
inbox: inbox/addw_face
|
||||
extra_train: []
|
||||
|
||||
isa:
|
||||
type: detect
|
||||
task_dir: isa
|
||||
nc: 4
|
||||
names: [indicative, prohibitory, warning, vehicle]
|
||||
inbox: inbox/isa
|
||||
extra_train: []
|
||||
|
||||
dam:
|
||||
type: detect
|
||||
domain: dms
|
||||
label: DAM 驾驶员监控
|
||||
type: multi
|
||||
task_dir: dam
|
||||
nc: 15
|
||||
names: [face, eye_open, eye_partially_open, eye_close, mouth_open, mouth_partially_open, mouth_close, side_face, nod_face, glasses, sunglasses, smoke, phone, driver, rise_face]
|
||||
inbox: inbox/dam
|
||||
extra_train: []
|
||||
modes:
|
||||
batch_0516:
|
||||
label: 0516 批次
|
||||
type: detect
|
||||
subdir: batch_0516
|
||||
nc: 15
|
||||
names: [face, eye_open, eye_partially_open, eye_close, mouth_open, mouth_partially_open, mouth_close, side_face, nod_face, glasses, sunglasses, smoke, phone, driver, rise_face]
|
||||
inbox: inbox/dam/batch_0516
|
||||
extra_train: []
|
||||
batch_0417:
|
||||
label: 0417 批次
|
||||
type: detect
|
||||
subdir: batch_0417
|
||||
nc: 15
|
||||
names: [face, eye_open, eye_partially_open, eye_close, mouth_open, mouth_partially_open, mouth_close, side_face, nod_face, glasses, sunglasses, smoke, phone, driver, rise_face]
|
||||
inbox: inbox/dam/batch_0417
|
||||
extra_train: []
|
||||
|
||||
dam_0417:
|
||||
forward:
|
||||
domain: forward
|
||||
label: 前向·交通标志
|
||||
type: multi
|
||||
task_dir: forward
|
||||
modes:
|
||||
detect:
|
||||
label: 粗检测（4 类）
|
||||
type: detect
|
||||
subdir: detect
|
||||
nc: 4
|
||||
names: [indicative, prohibitory, warning, vehicle]
|
||||
inbox: inbox/forward/detect
|
||||
extra_train: []
|
||||
classify:
|
||||
label: 细分类
|
||||
type: classify
|
||||
subdir: classify
|
||||
inbox: inbox/forward/classify
|
||||
extra_train: []
|
||||
|
||||
adas:
|
||||
domain: forward
|
||||
label: 前向·ADAS 7 类检测
|
||||
type: detect
|
||||
task_dir: dam_0417
|
||||
nc: 15
|
||||
names: [face, eye_open, eye_partially_open, eye_close, mouth_open, mouth_partially_open, mouth_close, side_face, nod_face, glasses, sunglasses, smoke, phone, driver, rise_face]
|
||||
inbox: inbox/dam_0417
|
||||
extra_train: []
|
||||
|
||||
isa_class:
|
||||
type: classify
|
||||
task_dir: isa_class
|
||||
inbox: inbox/isa_class
|
||||
extra_train: []
|
||||
task_dir: adas
|
||||
nc: 7
|
||||
names: [Pedestrain, Car, Truck, Bus, Motor-vehicles, Tricycle, cones]
|
||||
mode: detect
|
||||
inbox: inbox/adas
|
||||
|
||||
@@ -1,207 +1,8 @@
|
||||
|
||||
======================================================================
|
||||
【ddaw_1124】YOLO 检测 path: /home/chengfanglu/DATA/DMS/DATASET/gyp/ddaw_1124
|
||||
======================================================================
|
||||
|
||||
当前划分: train=3824 val=424 test=0 合计=4248
|
||||
类别数(nc): 9 类别ID: [0, 1, 2, 3, 4, 5, 6, 7, 8]
|
||||
|
||||
--- 当前：含该类别的图片数（按 split）---
|
||||
cls train val val% test 合计
|
||||
0 3072 348 10.2% 0 3420
|
||||
1 686 65 8.7% 0 751
|
||||
2 1827 210 10.3% 0 2037
|
||||
3 619 71 10.3% 0 690
|
||||
4 279 29 9.4% 0 308
|
||||
5 881 99 10.1% 0 980
|
||||
6 2415 273 10.2% 0 2688
|
||||
7 659 64 8.9% 0 723
|
||||
8 138 15 9.8% 0 153
|
||||
|
||||
当前 val 占比: min=8.7% max=10.3% 极差=1.7% (目标≈10%)
|
||||
✓ 当前各类 val 比例已较均衡
|
||||
|
||||
--- 分层重分后(模拟, val_ratio=0.1, seed=42) ---
|
||||
train=3823 val=425
|
||||
cls train val val%
|
||||
0 3074 346 10.1%
|
||||
1 674 77 10.3%
|
||||
2 1834 203 10.0%
|
||||
3 620 70 10.1%
|
||||
4 277 31 10.1%
|
||||
5 882 98 10.0%
|
||||
6 2423 265 9.9%
|
||||
7 651 72 10.0%
|
||||
8 138 15 9.8%
|
||||
|
||||
重分后 val 占比: min=9.8% max=10.3% 极差=0.4%
|
||||
✓ 分层后各类 val 比例均衡（极差≤2%）
|
||||
|
||||
======================================================================
|
||||
【addw_0523】YOLO 检测 path: /home/chengfanglu/DATA/DMS/DATASET/gyp/addw_0523
|
||||
======================================================================
|
||||
|
||||
当前划分: train=2375 val=594 test=0 合计=2969
|
||||
类别数(nc): 4 类别ID: [0, 1, 2, 3]
|
||||
|
||||
--- 当前：含该类别的图片数（按 split）---
|
||||
cls train val val% test 合计
|
||||
0 1367 370 21.3% 0 1737
|
||||
1 415 120 22.4% 0 535
|
||||
2 1018 229 18.4% 0 1247
|
||||
3 1812 445 19.7% 0 2257
|
||||
|
||||
当前 val 占比: min=18.4% max=22.4% 极差=4.1% (目标≈10%)
|
||||
⚠ 当前各类 val 比例不均衡
|
||||
|
||||
--- 分层重分后(模拟, val_ratio=0.1, seed=42) ---
|
||||
train=2671 val=298
|
||||
cls train val val%
|
||||
0 1565 172 9.9%
|
||||
1 481 54 10.1%
|
||||
2 1121 126 10.1%
|
||||
3 2029 228 10.1%
|
||||
|
||||
重分后 val 占比: min=9.9% max=10.1% 极差=0.2%
|
||||
✓ 分层后各类 val 比例均衡（极差≤2%）
|
||||
|
||||
======================================================================
|
||||
【isa_detect】YOLO 检测 path: /home/chengfanglu/DATA/DMS/DATASET/gyp/isa_detect
|
||||
======================================================================
|
||||
|
||||
当前划分: train=50480 val=9584 test=0 合计=60064
|
||||
类别数(nc): 4 类别ID: [0, 1, 2, 3]
|
||||
|
||||
--- 当前：含该类别的图片数（按 split）---
|
||||
cls train val val% test 合计
|
||||
0 15871 2971 15.8% 0 18842
|
||||
1 33355 7060 17.5% 0 40415
|
||||
2 7348 1204 14.1% 0 8552
|
||||
3 14715 2429 14.2% 0 17144
|
||||
|
||||
当前 val 占比: min=14.1% max=17.5% 极差=3.4% (目标≈10%)
|
||||
⚠ 当前各类 val 比例不均衡
|
||||
|
||||
--- 分层重分后(模拟, val_ratio=0.1, seed=42) ---
|
||||
train=54066 val=5940
|
||||
cls train val val%
|
||||
0 16883 1902 10.1%
|
||||
1 36344 4013 9.9%
|
||||
2 7693 855 10.0%
|
||||
3 15435 1707 10.0%
|
||||
|
||||
重分后 val 占比: min=9.9% max=10.1% 极差=0.2%
|
||||
✓ 分层后各类 val 比例均衡（极差≤2%）
|
||||
|
||||
======================================================================
|
||||
【dam_0516】YOLO 检测 path: /home/chengfanglu/DATA/DMS/DATASET/gyp/dam_0516
|
||||
======================================================================
|
||||
|
||||
当前划分: train=3780 val=1890 test=0 合计=5670
|
||||
类别数(nc): 15 类别ID: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
|
||||
|
||||
--- 当前：含该类别的图片数（按 split）---
|
||||
cls train val val% test 合计
|
||||
0 2582 1337 34.1% 0 3919
|
||||
1 726 373 33.9% 0 1099
|
||||
2 1670 829 33.2% 0 2499
|
||||
3 784 395 33.5% 0 1179
|
||||
4 266 136 33.8% 0 402
|
||||
5 1141 542 32.2% 0 1683
|
||||
6 2066 1050 33.7% 0 3116
|
||||
7 670 324 32.6% 0 994
|
||||
8 445 217 32.8% 0 662
|
||||
9 1088 528 32.7% 0 1616
|
||||
10 306 164 34.9% 0 470
|
||||
11 838 464 35.6% 0 1302
|
||||
12 883 489 35.6% 0 1372
|
||||
13 3700 1843 33.2% 0 5543
|
||||
14 360 151 29.5% 0 511
|
||||
|
||||
当前 val 占比: min=29.5% max=35.6% 极差=6.1% (目标≈10%)
|
||||
⚠ 当前各类 val 比例不均衡
|
||||
|
||||
--- 分层重分后(模拟, val_ratio=0.1, seed=42) ---
|
||||
train=5104 val=566
|
||||
cls train val val%
|
||||
0 3525 394 10.1%
|
||||
1 986 113 10.3%
|
||||
2 2240 259 10.4%
|
||||
3 1074 105 8.9%
|
||||
4 362 40 10.0%
|
||||
5 1497 186 11.1%
|
||||
6 2826 290 9.3%
|
||||
7 896 98 9.9%
|
||||
8 595 67 10.1%
|
||||
9 1473 143 8.8%
|
||||
10 423 47 10.0%
|
||||
11 1180 122 9.4%
|
||||
12 1241 131 9.5%
|
||||
13 4988 555 10.0%
|
||||
14 459 52 10.2%
|
||||
|
||||
重分后 val 占比: min=8.8% max=11.1% 极差=2.2%
|
||||
~ 分层后明显改善，少数类因样本极少会有小幅偏差
|
||||
|
||||
======================================================================
|
||||
【yoloface-0726】YOLO 检测 path: /home/chengfanglu/DATA/DMS/DATASET/gyp/yoloface-0726
|
||||
======================================================================
|
||||
|
||||
当前划分: train=4145 val=1464 test=0 合计=5609
|
||||
类别数(nc): 1 类别ID: [0]
|
||||
|
||||
--- 当前：含该类别的图片数（按 split）---
|
||||
cls train val val% test 合计
|
||||
0 4137 1462 26.1% 0 5599
|
||||
|
||||
当前 val 占比: min=26.1% max=26.1% 极差=0.0% (目标≈10%)
|
||||
✓ 当前各类 val 比例已较均衡
|
||||
|
||||
--- 分层重分后(模拟, val_ratio=0.1, seed=42) ---
|
||||
train=4970 val=551
|
||||
cls train val val%
|
||||
0 4960 551 10.0%
|
||||
|
||||
重分后 val 占比: min=10.0% max=10.0% 极差=0.0%
|
||||
✓ 分层后各类 val 比例均衡（极差≤2%）
|
||||
|
||||
======================================================================
|
||||
【isa_class_0116】分类(文件夹) path: /home/chengfanglu/DATA/DMS/DATASET/gyp/isa_class_0116
|
||||
======================================================================
|
||||
split: ['test', 'train'] 类别数: 92 总图片: 61747
|
||||
|
||||
--- 每类图片数（前20类 + 汇总）---
|
||||
class test train
|
||||
005 43 393
|
||||
010 55 501
|
||||
015 47 424
|
||||
020 67 604
|
||||
030 214 1933
|
||||
040 152 1369
|
||||
050 279 2513
|
||||
060 224 2024
|
||||
070 179 1614
|
||||
080 285 2572
|
||||
090 116 1045
|
||||
100 207 1866
|
||||
110 88 794
|
||||
120 156 1407
|
||||
130 47 429
|
||||
arrow_right 126 1138
|
||||
arrow_up 38 350
|
||||
bycycle 40 368
|
||||
dis_005 30 279
|
||||
dis_010 23 213
|
||||
(仅列前20类，共92类)
|
||||
|
||||
当前为 train/test 结构（无 val）；若需 val 请用 stratified_split.py classify --src-split train
|
||||
若从 train 按类划 10% val: val% min=8.3% max=11.8% 极差=3.4%
|
||||
|
||||
test: 92类 min=1 max=310 avg=67 合计=6133
|
||||
|
||||
train: 92类 min=17 max=2792 avg=604 合计=55614
|
||||
|
||||
======================================================================
|
||||
【dam_src_0417】原始 jpg+xml path: /home/chengfanglu/DATA/DMS/DATASET/gyp/dam_src_0417
|
||||
======================================================================
|
||||
src_data_0417_pick: jpg=2455 xml=2455 (无 YOLO 标签，待转换后按类划分)
|
||||
[adas]
|
||||
Pedestrain: 19730
|
||||
Car: 241764
|
||||
Truck: 87538
|
||||
Bus: 6804
|
||||
Motor-vehicles: 38378
|
||||
Tricycle: 5654
|
||||
cones: 6419
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# 模型版本：current=线上/基线，candidate=待晋级（train → eval → promote）
|
||||
# data_yaml 由 active_packs 生成，路径固定为 manifests/yaml_active/
|
||||
# 模型版本：current=线上/基线，candidate=待晋级
|
||||
# multi 任务 forward 按 mode 分子项；训练 yaml 为 forward__detect / forward__classify
|
||||
|
||||
schema: model-versions-v2
|
||||
schema: model-versions-v3
|
||||
|
||||
ddaw:
|
||||
type: detect
|
||||
@@ -24,30 +24,38 @@ addw_face:
|
||||
candidate: null
|
||||
last_eval: null
|
||||
|
||||
isa:
|
||||
dam__batch_0516:
|
||||
task: dam
|
||||
mode: batch_0516
|
||||
type: detect
|
||||
data_yaml: manifests/yaml_active/isa.yaml
|
||||
data_yaml: manifests/yaml_active/dam__batch_0516.yaml
|
||||
current: null
|
||||
candidate: null
|
||||
last_eval: null
|
||||
|
||||
dam:
|
||||
dam__batch_0417:
|
||||
task: dam
|
||||
mode: batch_0417
|
||||
type: detect
|
||||
data_yaml: manifests/yaml_active/dam.yaml
|
||||
data_yaml: manifests/yaml_active/dam__batch_0417.yaml
|
||||
current: null
|
||||
candidate: null
|
||||
last_eval: null
|
||||
|
||||
dam_0417:
|
||||
forward__detect:
|
||||
task: forward
|
||||
mode: detect
|
||||
type: detect
|
||||
data_yaml: manifests/yaml_active/dam_0417.yaml
|
||||
data_yaml: manifests/yaml_active/forward__detect.yaml
|
||||
current: null
|
||||
candidate: null
|
||||
last_eval: null
|
||||
|
||||
isa_class:
|
||||
forward__classify:
|
||||
task: forward
|
||||
mode: classify
|
||||
type: classify
|
||||
data_yaml: manifests/yaml_active/isa_class.yaml
|
||||
data_yaml: manifests/yaml_active/forward__classify.yaml
|
||||
current: null
|
||||
candidate: null
|
||||
last_eval: null
|
||||
|
||||
7
datasets/dms/manifests/yaml_active/dam__batch_0417.yaml
Normal file
7
datasets/dms/manifests/yaml_active/dam__batch_0417.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
# dam__batch_0417 — packs: dms_v1
|
||||
path: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/dam/batch_0417
|
||||
train: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/dam_0417/images/train
|
||||
val: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/dam_0417/images/val
|
||||
|
||||
nc: 15
|
||||
names: ["face", "eye_open", "eye_partially_open", "eye_close", "mouth_open", "mouth_partially_open", "mouth_close", "side_face", "nod_face", "glasses", "sunglasses", "smoke", "phone", "driver", "rise_face"]
|
||||
7
datasets/dms/manifests/yaml_active/dam__batch_0516.yaml
Normal file
7
datasets/dms/manifests/yaml_active/dam__batch_0516.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
# dam__batch_0516 — packs: dms_v1
|
||||
path: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/dam/batch_0516
|
||||
train: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/_dam_stash_0516/images/train
|
||||
val: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/_dam_stash_0516/images/val
|
||||
|
||||
nc: 15
|
||||
names: ["face", "eye_open", "eye_partially_open", "eye_close", "mouth_open", "mouth_partially_open", "mouth_close", "side_face", "nod_face", "glasses", "sunglasses", "smoke", "phone", "driver", "rise_face"]
|
||||
@@ -0,0 +1,5 @@
|
||||
# forward__classify — pack: dms_v1
|
||||
path: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/isa_class
|
||||
train: train
|
||||
val: val
|
||||
test: test
|
||||
7
datasets/dms/manifests/yaml_active/forward__detect.yaml
Normal file
7
datasets/dms/manifests/yaml_active/forward__detect.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
# forward__detect — packs: dms_v1
|
||||
path: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/forward/detect
|
||||
train: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/isa/images/train
|
||||
val: /home/chengfanglu/DATA/workspace/DMS/DATASET/packs/dms_v1/isa/images/val
|
||||
|
||||
nc: 4
|
||||
names: ["indicative", "prohibitory", "warning", "vehicle"]
|
||||
347
datasets/dms/scripts/export_ls_to_yolo.py
Normal file
347
datasets/dms/scripts/export_ls_to_yolo.py
Normal file
@@ -0,0 +1,347 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Label Studio ls_annotations JSON → YOLO detect / YOLO pose txt."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
DMS_ROOT = SCRIPT_DIR.parent
|
||||
CONFIG_DIR = DMS_ROOT / "configs"
|
||||
REGISTRY_PATH = DMS_ROOT / "datasets.registry.yaml"
|
||||
KPT_ORDER_DIR = CONFIG_DIR / "keypoint_order"
|
||||
|
||||
IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".JPG", ".JPEG", ".PNG"}
|
||||
ANNOTATIONS_DIRNAME = "ls_annotations"
|
||||
|
||||
|
||||
def _load_registry() -> dict[str, Any]:
|
||||
return yaml.safe_load(REGISTRY_PATH.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _resolve_task_config(task: str, mode: str | None = None) -> dict[str, Any]:
|
||||
from task_registry import get_mode_config, resolve_task_id
|
||||
|
||||
reg = _load_registry()
|
||||
task_r, mode_r = resolve_task_id(task, mode)
|
||||
return get_mode_config(task_r, mode_r, reg)
|
||||
|
||||
|
||||
def _class_name_to_id(names: list[str] | dict[int | str, str]) -> dict[str, int]:
|
||||
if isinstance(names, dict):
|
||||
return {str(v): int(k) for k, v in names.items()}
|
||||
return {name: idx for idx, name in enumerate(names)}
|
||||
|
||||
|
||||
def _load_kpt_label_map(task: str) -> dict[str, int]:
|
||||
path = KPT_ORDER_DIR / f"{task}_37.yaml"
|
||||
if not path.is_file():
|
||||
path = KPT_ORDER_DIR / f"{task}.yaml"
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"keypoint manifest not found for task {task}")
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
out: dict[str, int] = {}
|
||||
for item in data.get("keypoints") or []:
|
||||
out[str(item["label"])] = int(item["id"])
|
||||
return out
|
||||
|
||||
|
||||
def _task_id_for_image(image_path: Path, batch_dir: Path) -> str:
|
||||
try:
|
||||
rel = image_path.relative_to(batch_dir)
|
||||
stem = rel.as_posix()
|
||||
except ValueError:
|
||||
stem = image_path.stem
|
||||
return hashlib.sha256(stem.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def _iter_batch_images(batch_dir: Path) -> list[Path]:
|
||||
if not batch_dir.is_dir():
|
||||
return []
|
||||
candidates: list[Path] = []
|
||||
search_roots = [
|
||||
batch_dir / "images",
|
||||
batch_dir / "images" / "train",
|
||||
batch_dir,
|
||||
]
|
||||
seen: set[str] = set()
|
||||
for root in search_roots:
|
||||
if not root.is_dir():
|
||||
continue
|
||||
for p in sorted(root.rglob("*")):
|
||||
if not p.is_file() or p.suffix not in IMG_EXTS:
|
||||
continue
|
||||
key = str(p.resolve())
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
candidates.append(p.resolve())
|
||||
return candidates
|
||||
|
||||
|
||||
def _label_out_path(image_path: Path, batch_dir: Path) -> Path:
|
||||
try:
|
||||
rel = image_path.relative_to(batch_dir)
|
||||
except ValueError:
|
||||
rel = Path(image_path.name)
|
||||
parts = list(rel.parts)
|
||||
if parts and parts[0] == "images":
|
||||
parts = parts[1:]
|
||||
if parts and parts[0] in ("train", "val", "test"):
|
||||
split = parts[0]
|
||||
name = Path(*parts[1:]).with_suffix(".txt")
|
||||
return batch_dir / "labels" / split / name
|
||||
name = Path(*parts).with_suffix(".txt")
|
||||
return batch_dir / "labels" / name
|
||||
|
||||
|
||||
def _extract_result_regions(data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
result = data.get("result")
|
||||
if isinstance(result, list) and result:
|
||||
return result
|
||||
annotations = data.get("annotations")
|
||||
if isinstance(annotations, list) and annotations:
|
||||
first = annotations[0]
|
||||
if isinstance(first, dict) and isinstance(first.get("result"), list):
|
||||
return first["result"]
|
||||
return []
|
||||
|
||||
|
||||
def _clamp01(v: float) -> float:
|
||||
return max(0.0, min(1.0, v))
|
||||
|
||||
|
||||
def _ls_rect_to_yolo_bbox(value: dict[str, Any]) -> tuple[float, float, float, float]:
|
||||
x = float(value["x"])
|
||||
y = float(value["y"])
|
||||
w = float(value["width"])
|
||||
h = float(value["height"])
|
||||
cx = _clamp01((x + w / 2.0) / 100.0)
|
||||
cy = _clamp01((y + h / 2.0) / 100.0)
|
||||
nw = _clamp01(w / 100.0)
|
||||
nh = _clamp01(h / 100.0)
|
||||
return cx, cy, nw, nh
|
||||
|
||||
|
||||
def _ls_point_to_yolo_xy(value: dict[str, Any]) -> tuple[float, float]:
|
||||
return _clamp01(float(value["x"]) / 100.0), _clamp01(float(value["y"]) / 100.0)
|
||||
|
||||
|
||||
def _bbox_center(bbox: tuple[float, float, float, float]) -> tuple[float, float]:
|
||||
cx, cy, _, _ = bbox
|
||||
return cx, cy
|
||||
|
||||
|
||||
def _parse_rectangles(
|
||||
regions: list[dict[str, Any]],
|
||||
class_map: dict[str, int],
|
||||
) -> list[dict[str, Any]]:
|
||||
boxes: list[dict[str, Any]] = []
|
||||
for region in regions:
|
||||
if region.get("type") != "rectanglelabels":
|
||||
continue
|
||||
value = region.get("value") or {}
|
||||
labels = value.get("rectanglelabels") or []
|
||||
if not labels:
|
||||
continue
|
||||
label = str(labels[0])
|
||||
if label not in class_map:
|
||||
continue
|
||||
bbox = _ls_rect_to_yolo_bbox(value)
|
||||
boxes.append({"class_id": class_map[label], "bbox": bbox, "region_id": region.get("id")})
|
||||
return boxes
|
||||
|
||||
|
||||
def _parse_keypoints(
|
||||
regions: list[dict[str, Any]],
|
||||
kpt_map: dict[str, int],
|
||||
) -> list[dict[str, Any]]:
|
||||
points: list[dict[str, Any]] = []
|
||||
for region in regions:
|
||||
rtype = region.get("type")
|
||||
if rtype not in ("keypointlabels", "keypoint"):
|
||||
continue
|
||||
value = region.get("value") or {}
|
||||
labels = value.get("keypointlabels") or []
|
||||
if not labels:
|
||||
continue
|
||||
label = str(labels[0])
|
||||
if label not in kpt_map:
|
||||
continue
|
||||
x, y = _ls_point_to_yolo_xy(value)
|
||||
points.append({"index": kpt_map[label], "x": x, "y": y, "region_id": region.get("id")})
|
||||
return points
|
||||
|
||||
|
||||
def _assign_keypoints_to_boxes(
|
||||
boxes: list[dict[str, Any]],
|
||||
points: list[dict[str, Any]],
|
||||
) -> dict[int | None, list[dict[str, Any]]]:
|
||||
if not boxes:
|
||||
return {None: points}
|
||||
if len(boxes) == 1:
|
||||
return {0: points}
|
||||
|
||||
assigned: dict[int, list[dict[str, Any]]] = {i: [] for i in range(len(boxes))}
|
||||
for pt in points:
|
||||
best_i = 0
|
||||
best_d = float("inf")
|
||||
for i, box in enumerate(boxes):
|
||||
cx, cy = _bbox_center(box["bbox"])
|
||||
d = (pt["x"] - cx) ** 2 + (pt["y"] - cy) ** 2
|
||||
if d < best_d:
|
||||
best_d = d
|
||||
best_i = i
|
||||
assigned[best_i].append(pt)
|
||||
return assigned
|
||||
|
||||
|
||||
def _format_detect_line(class_id: int, bbox: tuple[float, float, float, float]) -> str:
|
||||
cx, cy, w, h = bbox
|
||||
return f"{class_id} {cx:.6f} {cy:.6f} {w:.6f} {h:.6f}"
|
||||
|
||||
|
||||
def _format_pose_line(
|
||||
class_id: int,
|
||||
bbox: tuple[float, float, float, float],
|
||||
points: list[dict[str, Any]],
|
||||
nk: int,
|
||||
) -> str:
|
||||
slots: list[tuple[float, float, float]] = [(0.0, 0.0, 0.0)] * nk
|
||||
for pt in points:
|
||||
idx = int(pt["index"])
|
||||
if 0 <= idx < nk:
|
||||
slots[idx] = (pt["x"], pt["y"], 2.0)
|
||||
parts = _format_detect_line(class_id, bbox).split()
|
||||
for x, y, v in slots:
|
||||
parts.extend([f"{x:.6f}", f"{y:.6f}", f"{v:.6f}"])
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def convert_regions_to_yolo_lines(
|
||||
regions: list[dict[str, Any]],
|
||||
*,
|
||||
mode: str,
|
||||
class_map: dict[str, int],
|
||||
kpt_map: dict[str, int] | None = None,
|
||||
kpt_shape: list[int] | None = None,
|
||||
) -> list[str]:
|
||||
if mode == "detect":
|
||||
lines = []
|
||||
for box in _parse_rectangles(regions, class_map):
|
||||
lines.append(_format_detect_line(box["class_id"], box["bbox"]))
|
||||
return lines
|
||||
|
||||
if mode != "pose":
|
||||
raise ValueError(f"unsupported mode: {mode}")
|
||||
if not kpt_map or not kpt_shape:
|
||||
raise ValueError("pose mode requires kpt_map and kpt_shape")
|
||||
|
||||
nk = int(kpt_shape[0])
|
||||
boxes = _parse_rectangles(regions, class_map)
|
||||
points = _parse_keypoints(regions, kpt_map)
|
||||
if not boxes:
|
||||
return []
|
||||
|
||||
assigned = _assign_keypoints_to_boxes(boxes, points)
|
||||
lines: list[str] = []
|
||||
for i, box in enumerate(boxes):
|
||||
pts = assigned.get(i, [])
|
||||
lines.append(_format_pose_line(box["class_id"], box["bbox"], pts, nk))
|
||||
return lines
|
||||
|
||||
|
||||
def export_batch(
|
||||
batch_dir: Path,
|
||||
task: str,
|
||||
*,
|
||||
mode: str,
|
||||
task_mode: str | None = None,
|
||||
out_subdir: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
batch_dir = batch_dir.resolve()
|
||||
tcfg = _resolve_task_config(task, task_mode)
|
||||
class_map = _class_name_to_id(tcfg.get("names") or {})
|
||||
kpt_map: dict[str, int] | None = None
|
||||
kpt_shape: list[int] | None = None
|
||||
if mode == "pose":
|
||||
kpt_map = _load_kpt_label_map(task)
|
||||
kpt_shape = list(tcfg.get("kpt_shape") or [37, 3])
|
||||
|
||||
ann_dir = batch_dir / "labels" / ANNOTATIONS_DIRNAME
|
||||
written = 0
|
||||
skipped_empty = 0
|
||||
missing_ann = 0
|
||||
|
||||
for image_path in _iter_batch_images(batch_dir):
|
||||
task_id = _task_id_for_image(image_path, batch_dir)
|
||||
ann_path = ann_dir / f"{task_id}.json"
|
||||
if not ann_path.is_file():
|
||||
missing_ann += 1
|
||||
continue
|
||||
data = json.loads(ann_path.read_text(encoding="utf-8"))
|
||||
regions = _extract_result_regions(data)
|
||||
if not regions:
|
||||
skipped_empty += 1
|
||||
continue
|
||||
lines = convert_regions_to_yolo_lines(
|
||||
regions,
|
||||
mode=mode,
|
||||
class_map=class_map,
|
||||
kpt_map=kpt_map,
|
||||
kpt_shape=kpt_shape,
|
||||
)
|
||||
if not lines:
|
||||
skipped_empty += 1
|
||||
continue
|
||||
out_path = _label_out_path(image_path, batch_dir)
|
||||
if out_subdir:
|
||||
# 显式覆盖：相对 batch_dir 的子目录 + 文件名
|
||||
out_path = batch_dir / out_subdir / f"{image_path.stem}.txt"
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
written += 1
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"batch_dir": str(batch_dir),
|
||||
"task": task,
|
||||
"mode": mode,
|
||||
"written": written,
|
||||
"skipped_empty": skipped_empty,
|
||||
"missing_ann": missing_ann,
|
||||
"out_subdir": out_subdir or "auto",
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Export Label Studio annotations to YOLO txt")
|
||||
parser.add_argument("--batch-dir", type=Path, required=True)
|
||||
parser.add_argument("--task", required=True)
|
||||
parser.add_argument("--mode", choices=("detect", "pose"), required=True)
|
||||
parser.add_argument("--task-mode", default=None, help="dam batch_0516 / batch_0417 等")
|
||||
parser.add_argument("--out-subdir", default="labels/train")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
result = export_batch(
|
||||
args.batch_dir,
|
||||
args.task,
|
||||
mode=args.mode,
|
||||
task_mode=args.task_mode,
|
||||
out_subdir=args.out_subdir,
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0 if result["written"] > 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -236,7 +236,7 @@ def ingest_yolo(
|
||||
|
||||
if is_voc:
|
||||
if tcfg["type"] != "detect":
|
||||
raise SystemExit("VOC xml 仅支持 detect 任务（dam / dam_0417）")
|
||||
raise SystemExit("VOC xml 仅支持 detect 任务（dam 各批次等）")
|
||||
staging = data_root.parent / "_staging_voc" / task
|
||||
staging_parent = staging.parent
|
||||
if staging.exists() and not dry_run:
|
||||
@@ -384,7 +384,11 @@ def ingest_one(
|
||||
src: Path,
|
||||
args: argparse.Namespace,
|
||||
) -> dict:
|
||||
tcfg = reg["tasks"][task]
|
||||
from task_registry import get_mode_config, resolve_task_id
|
||||
|
||||
submode = getattr(args, "mode", None) or getattr(args, "submode", None)
|
||||
task, submode = resolve_task_id(task, submode)
|
||||
tcfg = get_mode_config(task, submode, reg)
|
||||
pack = getattr(args, "pack", None) or "dms_v1"
|
||||
data_root = pack_task_data_root(root, pack, tcfg["task_dir"])
|
||||
sk = split_kwargs(reg, args)
|
||||
@@ -449,23 +453,42 @@ def ingest_all_sources(root: Path, reg: dict, task: str, args: argparse.Namespac
|
||||
|
||||
|
||||
def ingest_all_inbox(root: Path, reg: dict, args: argparse.Namespace) -> None:
|
||||
from task_registry import inbox_dir
|
||||
|
||||
pack = getattr(args, "pack", None) or "dms_v1"
|
||||
for task, tcfg in reg["tasks"].items():
|
||||
inbox = root / tcfg.get("inbox", f"inbox/{task}")
|
||||
if not inbox.is_dir():
|
||||
continue
|
||||
batches = sorted(d for d in inbox.iterdir() if d.is_dir())
|
||||
if not batches:
|
||||
continue
|
||||
print(f"\n>>> inbox {task}: {len(batches)} batch(es)")
|
||||
for batch in batches:
|
||||
ingest_one(root, reg, task, batch, args)
|
||||
if not args.dry_run:
|
||||
append_log(root, {"src": str(batch), "task": task, "pack": pack, "via": "inbox"})
|
||||
if tcfg.get("type") == "multi":
|
||||
for mode in (tcfg.get("modes") or {}):
|
||||
inbox = inbox_dir(root, task, mode, reg)
|
||||
if not inbox.is_dir():
|
||||
continue
|
||||
batches = sorted(d for d in inbox.iterdir() if d.is_dir())
|
||||
if not batches:
|
||||
continue
|
||||
print(f"\n>>> inbox {task}/{mode}: {len(batches)} batch(es)")
|
||||
args.submode = mode
|
||||
for batch in batches:
|
||||
ingest_one(root, reg, task, batch, args)
|
||||
if not args.dry_run:
|
||||
append_log(root, {"src": str(batch), "task": task, "mode": mode, "pack": pack, "via": "inbox"})
|
||||
else:
|
||||
inbox = inbox_dir(root, task, None, reg)
|
||||
if not inbox.is_dir():
|
||||
continue
|
||||
batches = sorted(d for d in inbox.iterdir() if d.is_dir())
|
||||
if not batches:
|
||||
continue
|
||||
print(f"\n>>> inbox {task}: {len(batches)} batch(es)")
|
||||
for batch in batches:
|
||||
ingest_one(root, reg, task, batch, args)
|
||||
if not args.dry_run:
|
||||
append_log(root, {"src": str(batch), "task": task, "pack": pack, "via": "inbox"})
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser(description="DMS 全任务增量接入")
|
||||
p.add_argument("--task", help="registry 任务名；与 --all-inbox 二选一")
|
||||
p.add_argument("--task", help="registry 任务名（forward 等 multi 任务需配合 --submode）")
|
||||
p.add_argument("--submode", choices=("detect", "classify"), help="multi 任务子模式，如 forward 的 detect/classify")
|
||||
p.add_argument("--src", type=Path, help="新数据目录")
|
||||
p.add_argument("--all-inbox", action="store_true", help="处理所有 inbox/<task>/* 批次")
|
||||
p.add_argument("--all-sources", action="store_true", help="处理任务 data/sources/* 下所有待合并批次")
|
||||
|
||||
77
datasets/dms/scripts/migrate_dam_layout.py
Normal file
77
datasets/dms/scripts/migrate_dam_layout.py
Normal file
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
"""将 dam / dam_0417 合并为 dam/batch_0516、dam/batch_0417（默认符号链接）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def link_or_move(src: Path, dst: Path, *, move: bool) -> None:
|
||||
if not src.is_dir():
|
||||
print(f" skip（不存在）: {src}")
|
||||
return
|
||||
if dst.exists():
|
||||
print(f" 已存在: {dst}")
|
||||
return
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if move:
|
||||
shutil.move(str(src), str(dst))
|
||||
print(f" moved {src} -> {dst}")
|
||||
else:
|
||||
dst.symlink_to(src.resolve())
|
||||
print(f" symlink {dst} -> {src.resolve()}")
|
||||
|
||||
|
||||
def migrate_pack(pack_dir: Path, *, move: bool) -> None:
|
||||
dam_root = pack_dir / "dam"
|
||||
dam_0417 = pack_dir / "dam_0417"
|
||||
batch_0516 = dam_root / "batch_0516"
|
||||
batch_0417 = dam_root / "batch_0417"
|
||||
|
||||
if batch_0516.exists() and batch_0417.exists():
|
||||
print(" dam 已迁移，跳过")
|
||||
return
|
||||
|
||||
# 当前 dam 为扁平 YOLO 布局（images/labels 在根下）
|
||||
if dam_root.is_dir() and (dam_root / "images").is_dir() and not batch_0516.exists():
|
||||
stash = pack_dir / "_dam_stash_0516"
|
||||
if stash.exists():
|
||||
print(f" 清理旧 stash: {stash}")
|
||||
if stash.is_symlink():
|
||||
stash.unlink()
|
||||
else:
|
||||
shutil.rmtree(stash)
|
||||
shutil.move(str(dam_root), str(stash))
|
||||
dam_root.mkdir(parents=True)
|
||||
link_or_move(stash, batch_0516, move=move)
|
||||
|
||||
if dam_0417.is_dir():
|
||||
link_or_move(dam_0417, batch_0417, move=move)
|
||||
|
||||
|
||||
def migrate_inbox(dms_root: Path, *, move: bool) -> None:
|
||||
for old, new in (
|
||||
("dam", "dam/batch_0516"),
|
||||
("dam_0417", "dam/batch_0417"),
|
||||
):
|
||||
src = dms_root / "inbox" / old
|
||||
dst = dms_root / "inbox" / new
|
||||
if src.is_dir():
|
||||
link_or_move(src, dst, move=move)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--pack-dir", type=Path, required=True)
|
||||
p.add_argument("--dms-root", type=Path, help="datasets/dms 根，迁移 inbox")
|
||||
p.add_argument("--move", action="store_true")
|
||||
args = p.parse_args()
|
||||
migrate_pack(args.pack_dir.resolve(), move=args.move)
|
||||
if args.dms_root:
|
||||
migrate_inbox(args.dms_root.resolve(), move=args.move)
|
||||
print("完成。请运行 refresh_yaml.py --task dam 并刷新 catalog。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
57
datasets/dms/scripts/migrate_forward_layout.py
Normal file
57
datasets/dms/scripts/migrate_forward_layout.py
Normal file
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""将 isa / isa_class 目录迁入 forward/detect、forward/classify（默认符号链接，保留原数据）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def link_or_move(src: Path, dst: Path, *, move: bool) -> None:
|
||||
if not src.is_dir():
|
||||
print(f" skip（不存在）: {src}")
|
||||
return
|
||||
if dst.exists():
|
||||
print(f" 已存在: {dst}")
|
||||
return
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if move:
|
||||
shutil.move(str(src), str(dst))
|
||||
print(f" moved {src} -> {dst}")
|
||||
else:
|
||||
dst.symlink_to(src.resolve())
|
||||
print(f" symlink {dst} -> {src.resolve()}")
|
||||
|
||||
|
||||
def migrate_pack(pack_dir: Path, *, move: bool) -> None:
|
||||
forward = pack_dir / "forward"
|
||||
forward.mkdir(parents=True, exist_ok=True)
|
||||
link_or_move(pack_dir / "isa", forward / "detect", move=move)
|
||||
link_or_move(pack_dir / "isa_class", forward / "classify", move=move)
|
||||
|
||||
|
||||
def migrate_inbox(dms_root: Path, *, move: bool) -> None:
|
||||
for old, new in (
|
||||
("isa", "forward/detect"),
|
||||
("isa_class", "forward/classify"),
|
||||
):
|
||||
src = dms_root / "inbox" / old
|
||||
dst = dms_root / "inbox" / new
|
||||
if src.is_dir():
|
||||
link_or_move(src, dst, move=move)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--pack-dir", type=Path, required=True, help="如 .../packs/dms_v1")
|
||||
p.add_argument("--dms-root", type=Path, help="datasets/dms 根，迁移 inbox")
|
||||
p.add_argument("--move", action="store_true", help="移动而非符号链接")
|
||||
args = p.parse_args()
|
||||
migrate_pack(args.pack_dir.resolve(), move=args.move)
|
||||
if args.dms_root:
|
||||
migrate_inbox(args.dms_root.resolve(), move=args.move)
|
||||
print("完成。请运行 refresh_yaml.py 并刷新平台 catalog。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,6 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""按 workflow active_packs 生成 manifests/yaml_active/*.yaml（可多包合并 train/val）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
@@ -11,10 +10,8 @@ import yaml
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
from pack_registry import ( # noqa: E402
|
||||
load_active_pack_names,
|
||||
resolve_pack_dir,
|
||||
)
|
||||
from pack_registry import load_active_pack_names, resolve_pack_dir # noqa: E402
|
||||
from task_registry import get_mode_config, load_registry, train_yaml_key # noqa: E402
|
||||
|
||||
|
||||
def fmt_names(names) -> str:
|
||||
@@ -39,13 +36,13 @@ def pack_task_root(root: Path, pack_name: str, task_dir: str) -> Path:
|
||||
|
||||
|
||||
def build_detect_pose_yaml(
|
||||
task: str,
|
||||
tcfg: dict,
|
||||
yaml_key: str,
|
||||
mcfg: dict,
|
||||
root: Path,
|
||||
pack_names: list[str],
|
||||
typ: str,
|
||||
) -> str:
|
||||
task_dir = tcfg["task_dir"]
|
||||
task_dir = mcfg["task_dir"]
|
||||
bases = []
|
||||
train_paths = []
|
||||
val_paths = []
|
||||
@@ -59,28 +56,28 @@ def build_detect_pose_yaml(
|
||||
val_paths.append(str((base / "images" / "val").resolve()))
|
||||
|
||||
if not bases:
|
||||
raise SystemExit(f"{task}: 无可用数据包目录")
|
||||
raise SystemExit(f"{yaml_key}: 无可用数据包目录")
|
||||
|
||||
lines = [
|
||||
f"# {task} — packs: {', '.join(pack_names)}",
|
||||
f"# {yaml_key} — packs: {', '.join(pack_names)}",
|
||||
f"path: {bases[0]}",
|
||||
yaml_list("train", train_paths),
|
||||
yaml_list("val", val_paths),
|
||||
"",
|
||||
]
|
||||
if typ == "pose":
|
||||
lines.insert(4, f"kpt_shape: {tcfg.get('kpt_shape', [37, 3])}")
|
||||
lines.insert(4, f"kpt_shape: {mcfg.get('kpt_shape', [37, 3])}")
|
||||
else:
|
||||
lines.extend([f"nc: {tcfg['nc']}", fmt_names(tcfg["names"]), ""])
|
||||
lines.extend([f"nc: {mcfg['nc']}", fmt_names(mcfg["names"]), ""])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_classify_yaml(task: str, tcfg: dict, root: Path, pack_names: list[str]) -> str:
|
||||
task_dir = tcfg["task_dir"]
|
||||
def build_classify_yaml(yaml_key: str, mcfg: dict, root: Path, pack_names: list[str]) -> str:
|
||||
task_dir = mcfg["task_dir"]
|
||||
if len(pack_names) > 1:
|
||||
print(f" warn {task}: classify 暂用首个包 {pack_names[0]}（多包请先合并目录）")
|
||||
print(f" warn {yaml_key}: classify 暂用首个包 {pack_names[0]}（多包请先合并目录）")
|
||||
base = pack_task_root(root, pack_names[0], task_dir)
|
||||
return f"""# {task} — pack: {pack_names[0]}
|
||||
return f"""# {yaml_key} — pack: {pack_names[0]}
|
||||
path: {base.resolve()}
|
||||
train: train
|
||||
val: val
|
||||
@@ -88,39 +85,49 @@ test: test
|
||||
"""
|
||||
|
||||
|
||||
def iter_yaml_jobs(reg: dict, only_task: str | None = None):
|
||||
tasks = load_registry(reg)
|
||||
if only_task:
|
||||
if only_task not in tasks:
|
||||
raise SystemExit(f"未知 task: {only_task}")
|
||||
tasks = {only_task: tasks[only_task]}
|
||||
for task, tcfg in tasks.items():
|
||||
if tcfg.get("type") == "multi":
|
||||
for mode in (tcfg.get("modes") or {}):
|
||||
mcfg = get_mode_config(task, mode, reg)
|
||||
key = train_yaml_key(task, mode, reg)
|
||||
yield key, mcfg
|
||||
else:
|
||||
yield task, tcfg
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--root", type=Path, default=SCRIPT_DIR.parent)
|
||||
p.add_argument("--packs", help="逗号分隔，覆盖 workflow active_packs")
|
||||
p.add_argument("--task", help="只生成某一任务")
|
||||
p.add_argument("--task", help="只生成某一任务（multi 会生成全部 mode）")
|
||||
args = p.parse_args()
|
||||
root = args.root.resolve()
|
||||
reg = yaml.safe_load((root / "datasets.registry.yaml").read_text(encoding="utf-8"))
|
||||
cli = [x.strip() for x in args.packs.split(",")] if args.packs else None
|
||||
pack_names = load_active_pack_names(root, cli)
|
||||
if not pack_names:
|
||||
raise SystemExit("active_packs 为空，请编辑 ML/workflow.registry.yaml 或 --packs")
|
||||
raise SystemExit("active_packs 为空，请编辑 workflow.registry.yaml 或 --packs")
|
||||
|
||||
out_dir = root / "manifests" / "yaml_active"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
print(f"active_packs: {pack_names}")
|
||||
|
||||
tasks = reg["tasks"]
|
||||
if args.task:
|
||||
if args.task not in tasks:
|
||||
raise SystemExit(f"未知 task: {args.task}")
|
||||
tasks = {args.task: tasks[args.task]}
|
||||
|
||||
for task, tcfg in tasks.items():
|
||||
typ = tcfg["type"]
|
||||
for yaml_key, mcfg in iter_yaml_jobs(reg, args.task):
|
||||
typ = mcfg["type"]
|
||||
if typ in ("detect", "pose"):
|
||||
content = build_detect_pose_yaml(task, tcfg, root, pack_names, typ)
|
||||
content = build_detect_pose_yaml(yaml_key, mcfg, root, pack_names, typ)
|
||||
elif typ == "classify":
|
||||
content = build_classify_yaml(task, tcfg, root, pack_names)
|
||||
content = build_classify_yaml(yaml_key, mcfg, root, pack_names)
|
||||
else:
|
||||
print(f" skip {task}: type {typ}")
|
||||
print(f" skip {yaml_key}: type {typ}")
|
||||
continue
|
||||
out = out_dir / f"{task}.yaml"
|
||||
out = out_dir / f"{yaml_key}.yaml"
|
||||
out.write_text(content, encoding="utf-8")
|
||||
print(f" wrote {out.relative_to(root)}")
|
||||
|
||||
|
||||
166
datasets/dms/scripts/task_registry.py
Normal file
166
datasets/dms/scripts/task_registry.py
Normal file
@@ -0,0 +1,166 @@
|
||||
"""DMS 任务注册表：domain 分组、multi 任务（前向 detect+classify）、旧 ID 别名。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
DOMAIN_LABELS = {
|
||||
"dms": "舱内 DMS",
|
||||
"forward": "前向 ADAS",
|
||||
}
|
||||
|
||||
# 报表 / 旧目录名 -> (task, mode)
|
||||
REPORT_TASK_ALIASES: dict[str, tuple[str, str | None]] = {
|
||||
"isa": ("forward", "detect"),
|
||||
"isa_detect": ("forward", "detect"),
|
||||
"isa_class": ("forward", "classify"),
|
||||
"isa_class_0116": ("forward", "classify"),
|
||||
"dam_0417": ("dam", "batch_0417"),
|
||||
}
|
||||
|
||||
LEGACY_TASK_ALIASES: dict[str, tuple[str, str | None]] = {
|
||||
"isa": ("forward", "detect"),
|
||||
"isa_class": ("forward", "classify"),
|
||||
"dam_0417": ("dam", "batch_0417"),
|
||||
}
|
||||
|
||||
|
||||
def load_registry(reg: dict) -> dict[str, Any]:
|
||||
return reg.get("tasks") or {}
|
||||
|
||||
|
||||
def resolve_task_id(task: str, mode: str | None = None) -> tuple[str, str | None]:
|
||||
"""用户/历史 task ID -> (canonical_task, mode)。"""
|
||||
if task in LEGACY_TASK_ALIASES:
|
||||
t, m = LEGACY_TASK_ALIASES[task]
|
||||
return t, mode or m
|
||||
return task, mode
|
||||
|
||||
|
||||
def report_task_key(task: str, mode: str | None = None) -> str:
|
||||
"""catalog 报表 CSV 中的任务列名。"""
|
||||
t, m = resolve_task_id(task, mode)
|
||||
if t == "forward" and m == "detect":
|
||||
return "isa"
|
||||
if t == "forward" and m == "classify":
|
||||
return "isa_class"
|
||||
if t == "dam" and m == "batch_0516":
|
||||
return "dam"
|
||||
if t == "dam" and m == "batch_0417":
|
||||
return "dam_0417"
|
||||
return task
|
||||
|
||||
|
||||
def train_yaml_key(task: str, mode: str | None, reg: dict) -> str:
|
||||
"""manifests/yaml_active 与 train.sh 使用的文件名（不含 .yaml）。"""
|
||||
task, mode = resolve_task_id(task, mode)
|
||||
tcfg = load_registry(reg)[task]
|
||||
if tcfg.get("type") == "multi":
|
||||
if not mode:
|
||||
raise ValueError(f"任务 {task} 需指定 mode（detect / classify）")
|
||||
return f"{task}__{mode}"
|
||||
return task
|
||||
|
||||
|
||||
def get_mode_config(task: str, mode: str | None, reg: dict) -> dict[str, Any]:
|
||||
task, mode = resolve_task_id(task, mode)
|
||||
tcfg = load_registry(reg)[task]
|
||||
if tcfg.get("type") != "multi":
|
||||
return {**tcfg, "task": task, "mode": None}
|
||||
modes = tcfg.get("modes") or {}
|
||||
if not mode:
|
||||
raise ValueError(f"任务 {task} 需指定 mode")
|
||||
if mode not in modes:
|
||||
raise ValueError(f"未知 mode: {task}/{mode}")
|
||||
mcfg = dict(modes[mode])
|
||||
mcfg["task"] = task
|
||||
mcfg["mode"] = mode
|
||||
mcfg["task_dir"] = f"{tcfg.get('task_dir', task)}/{mcfg.get('subdir', mode)}"
|
||||
mcfg["domain"] = tcfg.get("domain")
|
||||
mcfg["label"] = mcfg.get("label") or tcfg.get("label")
|
||||
return mcfg
|
||||
|
||||
|
||||
def task_data_dir(pack_dir, task: str, mode: str | None, reg: dict):
|
||||
from pathlib import Path
|
||||
|
||||
mcfg = get_mode_config(task, mode, reg)
|
||||
return Path(pack_dir) / mcfg["task_dir"]
|
||||
|
||||
|
||||
def inbox_dir(root, task: str, mode: str | None, reg: dict):
|
||||
from pathlib import Path
|
||||
|
||||
task, mode = resolve_task_id(task, mode)
|
||||
tcfg = load_registry(reg)[task]
|
||||
if tcfg.get("type") == "multi":
|
||||
mcfg = tcfg["modes"][mode or ""]
|
||||
rel = mcfg.get("inbox") or f"inbox/{task}/{mode}"
|
||||
return Path(root) / rel
|
||||
return Path(root) / (tcfg.get("inbox") or f"inbox/{task}")
|
||||
|
||||
|
||||
def iter_catalog_tasks(reg: dict) -> list[tuple[str, dict[str, Any]]]:
|
||||
"""catalog 顶层任务列表。"""
|
||||
out: list[tuple[str, dict[str, Any]]] = []
|
||||
for task, tcfg in load_registry(reg).items():
|
||||
entry = {
|
||||
"domain": tcfg.get("domain", "dms"),
|
||||
"domain_label": DOMAIN_LABELS.get(tcfg.get("domain", "dms"), tcfg.get("domain", "dms")),
|
||||
"label": tcfg.get("label", task),
|
||||
"type": tcfg.get("type"),
|
||||
}
|
||||
if tcfg.get("type") == "multi":
|
||||
entry["modes"] = {}
|
||||
for mode, mcfg in (tcfg.get("modes") or {}).items():
|
||||
entry["modes"][mode] = {
|
||||
"label": mcfg.get("label", mode),
|
||||
"type": mcfg.get("type"),
|
||||
"nc": mcfg.get("nc"),
|
||||
"names": mcfg.get("names"),
|
||||
"packs": [],
|
||||
"class_counts": {},
|
||||
}
|
||||
else:
|
||||
entry["nc"] = tcfg.get("nc")
|
||||
entry["names"] = tcfg.get("names")
|
||||
entry["packs"] = []
|
||||
entry["class_counts"] = {}
|
||||
out.append((task, entry))
|
||||
return out
|
||||
|
||||
|
||||
def map_report_task(report_name: str) -> tuple[str, str | None]:
|
||||
if report_name in REPORT_TASK_ALIASES:
|
||||
return REPORT_TASK_ALIASES[report_name]
|
||||
return report_name, None
|
||||
|
||||
|
||||
def task_defs_for_pending(reg: dict) -> dict[str, Any]:
|
||||
"""平台 pending API 的 task_defs。"""
|
||||
defs: dict[str, Any] = {}
|
||||
for task, tcfg in load_registry(reg).items():
|
||||
if tcfg.get("type") == "multi":
|
||||
defs[task] = {
|
||||
"type": "multi",
|
||||
"domain": tcfg.get("domain", "dms"),
|
||||
"label": tcfg.get("label", task),
|
||||
"modes": {
|
||||
m: {
|
||||
"type": mc.get("type"),
|
||||
"nc": mc.get("nc"),
|
||||
"names": mc.get("names"),
|
||||
"task_dir": f"{tcfg.get('task_dir', task)}/{mc.get('subdir', m)}",
|
||||
}
|
||||
for m, mc in (tcfg.get("modes") or {}).items()
|
||||
},
|
||||
}
|
||||
else:
|
||||
defs[task] = {
|
||||
"type": tcfg.get("type"),
|
||||
"domain": tcfg.get("domain", "dms"),
|
||||
"label": tcfg.get("label", task),
|
||||
"nc": tcfg.get("nc"),
|
||||
"names": tcfg.get("names"),
|
||||
"task_dir": tcfg.get("task_dir", task),
|
||||
}
|
||||
return defs
|
||||
143
datasets/dms/scripts/test_export_ls_to_yolo.py
Normal file
143
datasets/dms/scripts/test_export_ls_to_yolo.py
Normal file
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""export_ls_to_yolo 单元测试（无 pytest 依赖）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from export_ls_to_yolo import ( # noqa: E402
|
||||
convert_regions_to_yolo_lines,
|
||||
export_batch,
|
||||
)
|
||||
from ingest_incremental import validate_detect_label, validate_pose_label # noqa: E402
|
||||
|
||||
|
||||
def _task_id(rel: str) -> str:
|
||||
return hashlib.sha256(rel.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def test_detect_conversion() -> None:
|
||||
regions = [
|
||||
{
|
||||
"type": "rectanglelabels",
|
||||
"value": {
|
||||
"x": 10.0,
|
||||
"y": 20.0,
|
||||
"width": 30.0,
|
||||
"height": 40.0,
|
||||
"rectanglelabels": ["face"],
|
||||
},
|
||||
}
|
||||
]
|
||||
lines = convert_regions_to_yolo_lines(
|
||||
regions,
|
||||
mode="detect",
|
||||
class_map={"face": 0, "eye_open": 1},
|
||||
)
|
||||
assert len(lines) == 1
|
||||
parts = lines[0].split()
|
||||
assert len(parts) == 5
|
||||
assert parts[0] == "0"
|
||||
assert abs(float(parts[1]) - 0.25) < 1e-5 # cx = (10+15)/100
|
||||
assert abs(float(parts[2]) - 0.40) < 1e-5 # cy = (20+20)/100
|
||||
err = validate_detect_label("\n".join(lines), 4)
|
||||
assert err is None, err
|
||||
|
||||
|
||||
def test_pose_conversion() -> None:
|
||||
regions = [
|
||||
{
|
||||
"type": "rectanglelabels",
|
||||
"value": {
|
||||
"x": 10.0,
|
||||
"y": 20.0,
|
||||
"width": 30.0,
|
||||
"height": 40.0,
|
||||
"rectanglelabels": ["face"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "keypointlabels",
|
||||
"value": {"x": 35.6, "y": 52.9, "width": 0.5, "keypointlabels": ["kp_01"]},
|
||||
},
|
||||
{
|
||||
"type": "keypointlabels",
|
||||
"value": {"x": 50.0, "y": 50.0, "width": 0.5, "keypointlabels": ["kp_10"]},
|
||||
},
|
||||
]
|
||||
kpt_map = {f"kp_{i:02d}": i for i in range(37)}
|
||||
lines = convert_regions_to_yolo_lines(
|
||||
regions,
|
||||
mode="pose",
|
||||
class_map={"face": 0},
|
||||
kpt_map=kpt_map,
|
||||
kpt_shape=[37, 3],
|
||||
)
|
||||
assert len(lines) == 1
|
||||
parts = lines[0].split()
|
||||
assert len(parts) == 116
|
||||
assert parts[0] == "0"
|
||||
# kp_01 at index 1 -> fields 5+3..5+5
|
||||
assert abs(float(parts[8]) - 0.356) < 1e-3
|
||||
assert abs(float(parts[9]) - 0.529) < 1e-3
|
||||
assert parts[10] == "2.000000"
|
||||
err = validate_pose_label("\n".join(lines), [37, 3])
|
||||
assert err is None, err
|
||||
|
||||
|
||||
def test_export_batch_end_to_end() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
batch = Path(tmp)
|
||||
img_rel = "images/train/sample.jpg"
|
||||
img_path = batch / img_rel
|
||||
img_path.parent.mkdir(parents=True)
|
||||
img_path.write_bytes(b"\xff\xd8\xff")
|
||||
|
||||
tid = _task_id(img_rel)
|
||||
ann = {
|
||||
"task_id": tid,
|
||||
"result": [
|
||||
{
|
||||
"type": "rectanglelabels",
|
||||
"value": {
|
||||
"x": 10.0,
|
||||
"y": 20.0,
|
||||
"width": 30.0,
|
||||
"height": 40.0,
|
||||
"rectanglelabels": ["face"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "keypointlabels",
|
||||
"value": {"x": 25.0, "y": 40.0, "width": 0.5, "keypointlabels": ["kp_00"]},
|
||||
},
|
||||
],
|
||||
}
|
||||
ann_dir = batch / "labels" / "ls_annotations"
|
||||
ann_dir.mkdir(parents=True)
|
||||
(ann_dir / f"{tid}.json").write_text(json.dumps(ann), encoding="utf-8")
|
||||
|
||||
result = export_batch(batch, "addw_face", mode="pose")
|
||||
assert result["written"] == 1
|
||||
out = batch / "labels" / "train" / "sample.txt"
|
||||
assert out.is_file()
|
||||
parts = out.read_text().strip().split()
|
||||
assert len(parts) == 116
|
||||
|
||||
|
||||
def main() -> int:
|
||||
test_detect_conversion()
|
||||
test_pose_conversion()
|
||||
test_export_batch_end_to_end()
|
||||
print("OK export_ls_to_yolo tests")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -17,20 +17,21 @@ TASK="${1:?用法: $0 <task> [full|continue]}"
|
||||
TRAIN_MODE="${2:-full}"
|
||||
|
||||
REG="$DATASET_ROOT/datasets.registry.yaml"
|
||||
YAML="$DATASET_ROOT/manifests/yaml_active/${TASK}.yaml"
|
||||
VERSIONS="$DATASET_ROOT/manifests/train_versions.yaml"
|
||||
SUBMODE="${SUBMODE:-}"
|
||||
|
||||
if [[ ! -f "$YAML" ]]; then
|
||||
echo "找不到 yaml: $YAML"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
read -r TYPE MODE MODEL EPOCHS LR0 IMGSZ RUN_SUFFIX <<< "$(python3 - <<PY
|
||||
import yaml
|
||||
read -r YAML_KEY TYPE MODE MODEL EPOCHS LR0 IMGSZ RUN_SUFFIX <<< "$(python3 - <<PY
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
sys.path.insert(0, str(Path("$DATASET_ROOT/scripts")))
|
||||
from task_registry import get_mode_config, resolve_task_id, train_yaml_key
|
||||
reg = yaml.safe_load(Path("$REG").read_text())
|
||||
tcfg = reg["tasks"]["$TASK"]
|
||||
typ = tcfg["type"]
|
||||
task, sub = resolve_task_id("$TASK", "$SUBMODE" or None)
|
||||
mcfg = get_mode_config(task, sub, reg)
|
||||
typ = mcfg["type"]
|
||||
yaml_key = train_yaml_key(task, sub, reg)
|
||||
print(yaml_key, end=" ")
|
||||
train_mode = "$TRAIN_MODE" if "$TRAIN_MODE" in ("full", "continue") else reg.get("train", {}).get("mode", "full")
|
||||
t = reg.get("train", {}).get(typ, reg.get("train_defaults", {}).get(typ, {}))
|
||||
if train_mode == "continue":
|
||||
@@ -49,6 +50,12 @@ print(typ, mode, model, epochs, lr0, imgsz, suffix)
|
||||
PY
|
||||
)"
|
||||
|
||||
YAML="$DATASET_ROOT/manifests/yaml_active/${YAML_KEY}.yaml"
|
||||
if [[ ! -f "$YAML" ]]; then
|
||||
echo "找不到 yaml: $YAML（请先 refresh_yaml.py）"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# continue 模式：warm_start 为空则读 train_versions.yaml
|
||||
if [[ "$TRAIN_MODE" == "continue" && ( "$MODEL" == "null" || "$MODEL" == "None" || -z "$MODEL" ) ]]; then
|
||||
MODEL=$(python3 - <<PY 2>/dev/null || true
|
||||
@@ -57,7 +64,7 @@ from pathlib import Path
|
||||
p = Path("$VERSIONS")
|
||||
if p.is_file():
|
||||
v = yaml.safe_load(p.read_text()) or {}
|
||||
c = v.get("$TASK", {}).get("current")
|
||||
c = v.get("$YAML_KEY", {}).get("current")
|
||||
if c: print(c)
|
||||
PY
|
||||
)
|
||||
@@ -68,9 +75,9 @@ if [[ "$TRAIN_MODE" == "continue" && ( -z "$MODEL" || "$MODEL" == "null" ) ]]; t
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RUN_NAME="${TASK}_${RUN_SUFFIX}_$(date +%Y%m%d)"
|
||||
RUN_NAME="${YAML_KEY}_${RUN_SUFFIX}_$(date +%Y%m%d)"
|
||||
|
||||
echo "task=$TASK type=$TYPE yolo_mode=$MODE train_mode=$TRAIN_MODE"
|
||||
echo "task=$TASK submode=$SUBMODE yaml_key=$YAML_KEY type=$TYPE yolo_mode=$MODE train_mode=$TRAIN_MODE"
|
||||
echo "data=$YAML"
|
||||
echo "model=$MODEL epochs=$EPOCHS lr0=$LR0 imgsz=$IMGSZ name=$RUN_NAME"
|
||||
|
||||
@@ -93,4 +100,4 @@ yolo "$MODE" train \
|
||||
|
||||
BEST="runs/${MODE}/${RUN_NAME}/weights/best.pt"
|
||||
echo "完成: $BEST"
|
||||
echo "请更新 manifests/train_versions.yaml 中 $TASK.current = $BEST"
|
||||
echo "请更新 manifests/train_versions.yaml 中 $YAML_KEY.current = $BEST"
|
||||
|
||||
45
datasets/dms/versions/v1.json
Normal file
45
datasets/dms/versions/v1.json
Normal file
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"version_id": "v1",
|
||||
"project": "dms",
|
||||
"created_at": "2026-06-01T09:51:41.352091+00:00",
|
||||
"description": "初始版本",
|
||||
"author": "test",
|
||||
"parent_version": null,
|
||||
"summary": {
|
||||
"packs_count": 2,
|
||||
"total_images": 12814,
|
||||
"total_labels": 6000,
|
||||
"batches_count": 4
|
||||
},
|
||||
"packs": {
|
||||
"dms_v1": {
|
||||
"train_images": 11464,
|
||||
"val_images": 1350,
|
||||
"test_images": 0,
|
||||
"class_counts": {},
|
||||
"tasks": [
|
||||
"ddaw",
|
||||
"addw",
|
||||
"addw_face"
|
||||
]
|
||||
},
|
||||
"dms_v2": {
|
||||
"train_images": 0,
|
||||
"val_images": 0,
|
||||
"test_images": 0,
|
||||
"class_counts": {},
|
||||
"tasks": [
|
||||
"ddaw",
|
||||
"addw",
|
||||
"addw_face"
|
||||
]
|
||||
}
|
||||
},
|
||||
"batches": [
|
||||
"20260525_pilot",
|
||||
"addw_face",
|
||||
"ddaw",
|
||||
"labels"
|
||||
],
|
||||
"diff": null
|
||||
}
|
||||
50
datasets/dms/versions/v2.json
Normal file
50
datasets/dms/versions/v2.json
Normal file
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"version_id": "v2",
|
||||
"project": "dms",
|
||||
"created_at": "2026-06-01T09:51:51.218991+00:00",
|
||||
"description": "第二个版本",
|
||||
"author": "test",
|
||||
"parent_version": "v1",
|
||||
"summary": {
|
||||
"packs_count": 2,
|
||||
"total_images": 12814,
|
||||
"total_labels": 6000,
|
||||
"batches_count": 4
|
||||
},
|
||||
"packs": {
|
||||
"dms_v1": {
|
||||
"train_images": 11464,
|
||||
"val_images": 1350,
|
||||
"test_images": 0,
|
||||
"class_counts": {},
|
||||
"tasks": [
|
||||
"ddaw",
|
||||
"addw",
|
||||
"addw_face"
|
||||
]
|
||||
},
|
||||
"dms_v2": {
|
||||
"train_images": 0,
|
||||
"val_images": 0,
|
||||
"test_images": 0,
|
||||
"class_counts": {},
|
||||
"tasks": [
|
||||
"ddaw",
|
||||
"addw",
|
||||
"addw_face"
|
||||
]
|
||||
}
|
||||
},
|
||||
"batches": [
|
||||
"20260525_pilot",
|
||||
"addw_face",
|
||||
"ddaw",
|
||||
"labels"
|
||||
],
|
||||
"diff": {
|
||||
"added_packs": [],
|
||||
"removed_packs": [],
|
||||
"added_batches": [],
|
||||
"removed_batches": []
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user