feat: HSAP platform v2 — modular navigation, quality review, audit log, world model simulation
Major changes: - New frontend (platform/web/): Vite + React 18 + TypeScript + Tailwind - 4-module navigation: 数据送标 / 模型管理 / 车队管理 / 系统管理 - Data catalog with charts (DMS/ADAS/Lane 3-tab view) - Quality review workflow (标注质检): Good/Fine/Bad scoring with auto-advance - Audit enhancements: batch operations, rejection categories, Feishu notifications - Operation audit log (操作日志) - World model simulation studio (仿真工坊) - Dataset version management with snapshots and diff - ADAS 7-class dataset integration (138K images organized + compressed) - User management with Feishu integration and pagination - CRUD/search/filter on all pages, card layout redesign - PIL-optimized image overlay rendering - Auto-snapshot on build, in_review workflow stage - Removed embedded algorithm code (now in workspace)
This commit is contained in:
7
scripts/build_hsap_ls_ui.sh
Executable file
7
scripts/build_hsap_ls_ui.sh
Executable file
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# HSAP 前端构建（已迁移至 platform/web/ Vite 独立项目）
|
||||
# 本脚本保留作为向后兼容入口，实际构建委托给 build_web.sh
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
echo "[build] 使用新构建系统 platform/web/ (Vite)..."
|
||||
exec bash "$ROOT/scripts/build_web.sh"
|
||||
60
scripts/build_web.sh
Executable file
60
scripts/build_web.sh
Executable file
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build HSAP standalone frontend from platform/web/ and deploy to platform/ui-hsap/dist/
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
echo "[web] Installing dependencies..."
|
||||
cd "$ROOT/platform/web"
|
||||
npm ci --silent
|
||||
|
||||
echo "[web] Building..."
|
||||
npm run build
|
||||
|
||||
DIST="$ROOT/platform/ui-hsap/dist"
|
||||
|
||||
# Copy Label Studio Editor legacy build for /annotate/
|
||||
EDITOR_SRC="${LABEL_STUDIO_DIST:-${ROOT}/../workspace/BK2/label-studio/web/dist/apps/hsap-platform}"
|
||||
if [ -d "$EDITOR_SRC" ]; then
|
||||
mkdir -p "$DIST/annotate"
|
||||
cp -r "$EDITOR_SRC"/* "$DIST/annotate/"
|
||||
|
||||
# Copy webpack chunks to dist root (webpack publicPath="/" loads them from root)
|
||||
for f in editor.js 849.js 710.js 408.js 63.js main.css editor.css; do
|
||||
[ -f "$DIST/annotate/$f" ] && cp "$DIST/annotate/$f" "$DIST/$f"
|
||||
done
|
||||
|
||||
# Custom annotate index.html
|
||||
cat > "$DIST/annotate/index.html" << 'HTMLEOF'
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>HSAP · 标注编辑器</title>
|
||||
<base href="/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<link rel="stylesheet" href="/annotate/main.css" />
|
||||
<link rel="stylesheet" href="/annotate/editor.css" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script>
|
||||
(function() {
|
||||
var p = new URLSearchParams(location.search);
|
||||
var cid = p.get('c');
|
||||
if (cid) {
|
||||
history.replaceState(null, '', '/labeling/campaigns/' + cid + '/annotate');
|
||||
}
|
||||
})();
|
||||
</script>
|
||||
<script src="/annotate/main.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
HTMLEOF
|
||||
echo "[web] Annotate editor (legacy LS) deployed from $EDITOR_SRC"
|
||||
else
|
||||
echo "[web] ⚠ annotate editor source not found at $EDITOR_SRC"
|
||||
fi
|
||||
|
||||
echo "[web] Build complete → $DIST"
|
||||
echo "[web] Files:"
|
||||
ls -lh "$DIST/index.html" "$DIST/assets/" | head -8
|
||||
@@ -8,6 +8,11 @@ if [[ ! -f .env ]]; then
|
||||
cp .env.example .env
|
||||
echo "已创建 .env（默认挂载本仓库；大文件数据可设置 AS_WORKSPACE_ROOT）"
|
||||
fi
|
||||
DEFAULT_WS="$(cd "$ROOT/.." && pwd)/workspace"
|
||||
if [[ -d "$DEFAULT_WS/DMS" ]] && ! grep -q '^AS_WORKSPACE_ROOT=' .env 2>/dev/null; then
|
||||
echo "AS_WORKSPACE_ROOT=$DEFAULT_WS" >> .env
|
||||
echo "已写入 AS_WORKSPACE_ROOT=$DEFAULT_WS"
|
||||
fi
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "未安装 Docker。Ubuntu: sudo apt install docker.io docker-compose-v2"
|
||||
|
||||
39
scripts/feishu_bitable_verify.sh
Executable file
39
scripts/feishu_bitable_verify.sh
Executable file
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# 检查飞书多维表格配置与 HSAP 连通性（需平台已启动、feishu.env 已填 BITABLE_*）
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
if [[ -f manifests/feishu.env ]]; then
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source manifests/feishu.env
|
||||
set +a
|
||||
fi
|
||||
|
||||
BASE="${AS_FRONTEND_URL:-http://127.0.0.1:8787}"
|
||||
TOKEN="${HSAP_TOKEN:-}"
|
||||
|
||||
if [[ -z "$TOKEN" ]]; then
|
||||
if [[ "${AS_DEV_AUTH:-}" =~ ^(1|true|yes)$ ]]; then
|
||||
echo "==> 开发登录获取 token"
|
||||
TOKEN="$(curl -sS -X POST "$BASE/api/v1/auth/dev/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name":"feishu-verify"}' | python3 -c "import sys,json; print(json.load(sys.stdin).get('access_token',''))")"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$TOKEN" ]]; then
|
||||
echo "请设置 HSAP_TOKEN 或开启 AS_DEV_AUTH 后重试" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> GET $BASE/api/v1/integrations/feishu/bitable/status"
|
||||
curl -sS -H "Authorization: Bearer $TOKEN" "$BASE/api/v1/integrations/feishu/bitable/status" | python3 -m json.tool
|
||||
|
||||
echo ""
|
||||
echo "==> GET bitable/tables（查 table_id）"
|
||||
curl -sS -H "Authorization: Bearer $TOKEN" "$BASE/api/v1/integrations/feishu/bitable/tables" | python3 -m json.tool
|
||||
|
||||
echo ""
|
||||
echo "OK: 若 status.ok=true 且 missing_columns 为空，则列名与飞书表一致。"
|
||||
80
scripts/lake_checklist_audit.py
Executable file
80
scripts/lake_checklist_audit.py
Executable file
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""对照 DATA_LAKE_CHECKLIST 阶段 A～E，输出 HSAP 当前实现缺口（只读审计）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "platform"))
|
||||
|
||||
from as_platform.config import WORKSPACE # noqa: E402
|
||||
|
||||
|
||||
def _check(name: str, ok: bool, detail: str = "") -> dict:
|
||||
return {"item": name, "ok": ok, "detail": detail}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
checks: list[dict] = []
|
||||
|
||||
staging = WORKSPACE / "lake" / "staging"
|
||||
reports = WORKSPACE / "manifests" / "lake" / "reports"
|
||||
curated = WORKSPACE / "lake" / "curated"
|
||||
|
||||
checks.append(_check("A_staging_dir", staging.is_dir(), str(staging)))
|
||||
checks.append(
|
||||
_check(
|
||||
"A_upload_api",
|
||||
(ROOT / "platform/as_platform/data/lake.py").is_file(),
|
||||
"analyze_uploaded_candidate / promote",
|
||||
)
|
||||
)
|
||||
checks.append(
|
||||
_check(
|
||||
"B_analyze_job",
|
||||
(ROOT / "platform/as_platform/jobs/runner.py").is_file(),
|
||||
"analyze_uploaded_dataset action",
|
||||
)
|
||||
)
|
||||
checks.append(_check("B_reports_dir", reports.is_dir(), str(reports)))
|
||||
report_files = list(reports.glob("*.json")) if reports.is_dir() else []
|
||||
checks.append(_check("B_sample_report", len(report_files) > 0, f"count={len(report_files)}"))
|
||||
|
||||
checks.append(
|
||||
_check(
|
||||
"C_approval_flow",
|
||||
(ROOT / "platform/as_platform/audit/queue.py").is_file(),
|
||||
"delivery_ingest + approvals",
|
||||
)
|
||||
)
|
||||
checks.append(
|
||||
_check(
|
||||
"D_curated_dir",
|
||||
True,
|
||||
"optional until first promote" + ("" if curated.is_dir() else f" (missing {curated})"),
|
||||
)
|
||||
)
|
||||
checks.append(
|
||||
_check(
|
||||
"D_catalog_api",
|
||||
(ROOT / "platform/as_platform/api/server.py").is_file(),
|
||||
"GET /api/v1/catalog/*",
|
||||
)
|
||||
)
|
||||
|
||||
failed = [c for c in checks if not c["ok"]]
|
||||
out = {"workspace": str(WORKSPACE), "checks": checks, "failed_count": len(failed)}
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
if failed:
|
||||
print("\nLAKE_CHECKLIST_GAPS:", file=sys.stderr)
|
||||
for c in failed:
|
||||
print(f" - {c['item']}: {c['detail']}", file=sys.stderr)
|
||||
return 1
|
||||
print("LAKE_CHECKLIST_AUDIT_OK")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
201
scripts/organize_adas.py
Normal file
201
scripts/organize_adas.py
Normal file
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""整理 ADAS 数据集到 HSAP 平台可读格式。
|
||||
|
||||
ADAS 数据集目录结构:
|
||||
road_datas/wf_batch*/images/ + labels/ ← 标准格式，直接用
|
||||
OPEN/ONCE/tvt/{train,val,test}/.../images/ ← 需整理
|
||||
VAL_s/.../images/ ← 需整理
|
||||
|
||||
输出: datasets/dms/packs/adas_v1/ 下创建组织好的目录 + 生成 class summary。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ARCHIVE = Path("/home/chengfanglu/DATA/workspace/BK2/archive/adas_2d_det_dataset")
|
||||
DEST = Path("/home/chengfanglu/DATA/HSAP/datasets/dms/packs/adas_v1")
|
||||
MANIFESTS = Path("/home/chengfanglu/DATA/HSAP/datasets/dms/manifests")
|
||||
|
||||
CLASS_NAMES = ["Pedestrain", "Car", "Truck", "Bus", "Motor-vehicles", "Tricycle", "cones"]
|
||||
|
||||
|
||||
def count_yolo_labels(label_dir: Path) -> dict[int, int]:
|
||||
"""统计 YOLO label 目录中各类别实例数。"""
|
||||
counts: dict[int, int] = defaultdict(int)
|
||||
if not label_dir.is_dir():
|
||||
return dict(counts)
|
||||
for txt in label_dir.glob("*.txt"):
|
||||
try:
|
||||
for line in txt.read_text().strip().splitlines():
|
||||
parts = line.strip().split()
|
||||
if parts:
|
||||
cls_id = int(float(parts[0]))
|
||||
counts[cls_id] += 1
|
||||
except Exception:
|
||||
pass
|
||||
return dict(counts)
|
||||
|
||||
|
||||
def count_images(img_dir: Path) -> int:
|
||||
if not img_dir.is_dir():
|
||||
return 0
|
||||
return len([f for f in img_dir.iterdir() if f.suffix.lower() in {".jpg", ".jpeg", ".png", ".bmp"}])
|
||||
|
||||
|
||||
def organize_wf_batches() -> tuple[int, dict[int, int], int]:
|
||||
"""整理 road_datas/wf_batch*/ 到 adas/sources/ 。返回 (total_images, class_counts, total_boxes)。"""
|
||||
sources_dir = DEST / "adas" / "sources"
|
||||
sources_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
total_imgs = 0
|
||||
total_boxes = 0
|
||||
class_counts: dict[int, int] = defaultdict(int)
|
||||
|
||||
road_datas = ARCHIVE / "road_datas"
|
||||
if not road_datas.is_dir():
|
||||
print(f" ⚠ road_datas not found at {road_datas}")
|
||||
return 0, {}, 0
|
||||
|
||||
for batch_dir in sorted(road_datas.iterdir()):
|
||||
if not batch_dir.is_dir():
|
||||
continue
|
||||
img_dir = batch_dir / "images"
|
||||
lbl_dir = batch_dir / "labels"
|
||||
if not img_dir.is_dir():
|
||||
continue
|
||||
|
||||
batch_name = batch_dir.name
|
||||
dest_batch = sources_dir / batch_name
|
||||
if dest_batch.exists():
|
||||
continue # 已整理过
|
||||
|
||||
n_imgs = count_images(img_dir)
|
||||
if n_imgs == 0:
|
||||
continue
|
||||
|
||||
dest_batch.mkdir(parents=True, exist_ok=True)
|
||||
dest_img = dest_batch / "images"
|
||||
dest_lbl = dest_batch / "labels"
|
||||
|
||||
# 使用 symlink 节省磁盘空间
|
||||
dest_img.symlink_to(img_dir.resolve())
|
||||
if lbl_dir.is_dir():
|
||||
dest_lbl.symlink_to(lbl_dir.resolve())
|
||||
cc = count_yolo_labels(lbl_dir)
|
||||
for cls_id, cnt in cc.items():
|
||||
class_counts[cls_id] += cnt
|
||||
total_boxes += cnt
|
||||
|
||||
total_imgs += n_imgs
|
||||
print(f" ✓ {batch_name}: {n_imgs} imgs, {sum(cc.values()) if lbl_dir.is_dir() else 0} boxes")
|
||||
|
||||
return total_imgs, dict(class_counts), total_boxes
|
||||
|
||||
|
||||
def organize_once() -> tuple[int, dict[int, int], int]:
|
||||
"""整理 OPEN/ONCE/tvt/ 到 adas/sources/once_*/ 。"""
|
||||
once_dir = ARCHIVE / "OPEN" / "ONCE" / "tvt"
|
||||
if not once_dir.is_dir():
|
||||
print(f" ⚠ ONCE not found at {once_dir}")
|
||||
return 0, {}, 0
|
||||
|
||||
total_imgs = 0
|
||||
total_boxes = 0
|
||||
class_counts: dict[int, int] = defaultdict(int)
|
||||
sources_dir = DEST / "adas" / "sources"
|
||||
|
||||
for split in ["train", "val", "test"]:
|
||||
split_dir = once_dir / split
|
||||
if not split_dir.is_dir():
|
||||
continue
|
||||
for cam_dir in sorted(split_dir.iterdir()):
|
||||
if not cam_dir.is_dir():
|
||||
continue
|
||||
for scene_dir in sorted(cam_dir.iterdir()):
|
||||
if not scene_dir.is_dir():
|
||||
continue
|
||||
img_dir = scene_dir / "images"
|
||||
if not img_dir.is_dir():
|
||||
continue
|
||||
n_imgs = count_images(img_dir)
|
||||
if n_imgs == 0:
|
||||
continue
|
||||
|
||||
batch_name = f"once_{split}_{cam_dir.name}_{scene_dir.name}"
|
||||
dest_batch = sources_dir / batch_name
|
||||
if dest_batch.exists():
|
||||
continue
|
||||
|
||||
dest_batch.mkdir(parents=True, exist_ok=True)
|
||||
(dest_batch / "images").symlink_to(img_dir.resolve())
|
||||
total_imgs += n_imgs
|
||||
print(f" ✓ {batch_name}: {n_imgs} imgs")
|
||||
|
||||
return total_imgs, dict(class_counts), total_boxes
|
||||
|
||||
|
||||
def write_class_summary(total_boxes: dict[int, int]):
|
||||
"""生成平台可读的 class summary 文件。"""
|
||||
MANIFESTS.mkdir(parents=True, exist_ok=True)
|
||||
summary_path = MANIFESTS / "dataset_class_summary.txt"
|
||||
|
||||
# 读取已有内容（保留其他任务的统计）
|
||||
existing: dict[str, str] = {}
|
||||
if summary_path.is_file():
|
||||
current_task = None
|
||||
for line in summary_path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("[") and line.endswith("]"):
|
||||
current_task = line[1:-1]
|
||||
existing[current_task] = ""
|
||||
elif current_task:
|
||||
existing[current_task] += line + "\n"
|
||||
|
||||
# 生成 adas 统计
|
||||
lines = ["[adas]"]
|
||||
for cls_id in sorted(total_boxes.keys()):
|
||||
name = CLASS_NAMES[cls_id] if cls_id < len(CLASS_NAMES) else f"class_{cls_id}"
|
||||
lines.append(f"{name}: {total_boxes[cls_id]}")
|
||||
existing["adas"] = "\n".join(lines[1:]) + "\n"
|
||||
|
||||
# 写回
|
||||
with open(summary_path, "w") as f:
|
||||
for task, content in existing.items():
|
||||
f.write(f"[{task}]\n{content}")
|
||||
print(f" ✓ Class summary written to {summary_path}")
|
||||
|
||||
|
||||
def main():
|
||||
print("=== 整理 ADAS 数据集 ===")
|
||||
DEST.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print("\n1. 整理 wf_batch 批次...")
|
||||
wf_imgs, wf_classes, wf_boxes = organize_wf_batches()
|
||||
|
||||
print("\n2. 整理 ONCE 数据...")
|
||||
once_imgs, once_classes, once_boxes = organize_once()
|
||||
|
||||
# 合并统计
|
||||
total_boxes: dict[int, int] = defaultdict(int)
|
||||
for cls_id, cnt in wf_classes.items():
|
||||
total_boxes[cls_id] += cnt
|
||||
for cls_id, cnt in once_classes.items():
|
||||
total_boxes[cls_id] += cnt
|
||||
|
||||
print(f"\n=== 整理完成 ===")
|
||||
print(f" wf_batch 图片: {wf_imgs}, 标注框: {wf_boxes}")
|
||||
print(f" ONCE 图片: {once_imgs}")
|
||||
print(f" 总标注框: {sum(total_boxes.values())}")
|
||||
print(f" 各类别分布: {dict(total_boxes)}")
|
||||
|
||||
print("\n3. 生成 class summary...")
|
||||
write_class_summary(dict(total_boxes))
|
||||
|
||||
print("\n✅ 完成！可以刷新 catalog 了")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -6,6 +6,13 @@ set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
export PYTHONPATH="${ROOT}/platform${PYTHONPATH:+:$PYTHONPATH}"
|
||||
|
||||
if [[ -f "$ROOT/.env" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source "$ROOT/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
if [[ -f "$ROOT/manifests/feishu.env" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
@@ -15,6 +22,11 @@ fi
|
||||
|
||||
export AS_JOB_EXECUTOR="${AS_JOB_EXECUTOR:-thread}"
|
||||
|
||||
# .env 中 AS_REDIS_PORT=6380 时与 feishu.env 默认 6379 对齐
|
||||
if [[ -n "${AS_REDIS_PORT:-}" ]]; then
|
||||
export AS_REDIS_URL="redis://127.0.0.1:${AS_REDIS_PORT}/0"
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
bash scripts/setup_links.sh 2>/dev/null || true
|
||||
|
||||
|
||||
@@ -29,8 +29,11 @@ link_dir() {
|
||||
rm -f "$link"
|
||||
fi
|
||||
mkdir -p "$(dirname "$link")"
|
||||
ln -sfn "$target" "$link"
|
||||
echo " $link -> $target"
|
||||
local rel link_parent
|
||||
link_parent="$(dirname "$link")"
|
||||
rel="$(python3 -c "import os.path; print(os.path.relpath('''$target''', '''$link_parent'''))")"
|
||||
ln -sfn "$rel" "$link"
|
||||
echo " $link -> $rel"
|
||||
}
|
||||
|
||||
mkdir -p "$WS/Lane" "$ROOT/datasets" "$ROOT/algorithms/dms_yolo" "$ROOT/algorithms/lane_ufld"
|
||||
@@ -39,8 +42,22 @@ mkdir -p "$WS/Lane" "$ROOT/datasets" "$ROOT/algorithms/dms_yolo" "$ROOT/algorith
|
||||
[[ -d "$WS/lane" ]] && ln -sfn "$WS/lane" "$WS/Lane/dataset" 2>/dev/null || true
|
||||
[[ -d "$WS/LaneDection/Code" ]] && ln -sfn "$WS/LaneDection/Code" "$WS/Lane/code" 2>/dev/null || true
|
||||
|
||||
echo ">>> datasets"
|
||||
link_dir "$WS/DMS/DATASET" "$ROOT/datasets/dms"
|
||||
echo ">>> datasets/dms（保留仓库内 registry/scripts，仅软链 packs）"
|
||||
DMS_ROOT="$ROOT/datasets/dms"
|
||||
if [[ -L "$DMS_ROOT" ]]; then
|
||||
rm -f "$DMS_ROOT"
|
||||
fi
|
||||
if [[ ! -d "$DMS_ROOT" ]] && [[ -d "${DMS_ROOT}.embedded.bak" ]]; then
|
||||
mv "${DMS_ROOT}.embedded.bak" "$DMS_ROOT"
|
||||
fi
|
||||
mkdir -p "$DMS_ROOT/packs"
|
||||
if [[ -d "$WS/DMS/DATASET/packs" ]]; then
|
||||
for pack in "$WS/DMS/DATASET/packs"/*; do
|
||||
[[ -e "$pack" ]] || continue
|
||||
name="$(basename "$pack")"
|
||||
link_dir "$pack" "$DMS_ROOT/packs/$name"
|
||||
done
|
||||
fi
|
||||
link_dir "$WS/lane" "$ROOT/datasets/lane"
|
||||
|
||||
echo ">>> algorithms"
|
||||
|
||||
73
scripts/smoke_labeling_api.sh
Executable file
73
scripts/smoke_labeling_api.sh
Executable file
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
# 标注 API 冒烟：离线单元测试始终运行；API 部分需 platform 在 :8787（可用 HSAP_API_SKIP=1 跳过）
|
||||
set -euo pipefail
|
||||
BASE="${HSAP_API:-http://127.0.0.1:8787}"
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
echo "==> offline export_ls_to_yolo unit tests"
|
||||
python3 "$ROOT/datasets/dms/scripts/test_export_ls_to_yolo.py"
|
||||
|
||||
echo "==> offline export_ls_to_lane_gt unit tests"
|
||||
python3 "$ROOT/datasets/lane/scripts/test_export_ls_to_lane_gt.py"
|
||||
|
||||
if [[ "${HSAP_API_SKIP:-0}" == "1" ]]; then
|
||||
echo "SKIP API tests (HSAP_API_SKIP=1)"
|
||||
echo "OK (offline only)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "==> check platform $BASE"
|
||||
LOGIN_RESP=$(curl -sS -m 5 -X POST "$BASE/api/v1/auth/dev/login" \
|
||||
-H 'Content-Type: application/json' -d '{"name":"smoke"}' 2>&1) || LOGIN_RESP=""
|
||||
|
||||
TOKEN=$(printf '%s' "$LOGIN_RESP" | python3 -c "
|
||||
import sys, json
|
||||
raw = sys.stdin.read().strip()
|
||||
if not raw:
|
||||
sys.exit(1)
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
sys.exit(1)
|
||||
tok = data.get('access_token')
|
||||
if not tok:
|
||||
print(data.get('detail') or data, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(tok)
|
||||
" 2>/dev/null) || {
|
||||
echo "SKIP API tests: platform 未就绪 ($BASE)"
|
||||
echo " 启动: cd $ROOT && docker compose up -d platform"
|
||||
echo " 或: bash $ROOT/scripts/run_local.sh"
|
||||
echo " 仅跑离线测试可: HSAP_API_SKIP=1 bash $0"
|
||||
echo "OK (offline only)"
|
||||
exit 0
|
||||
}
|
||||
|
||||
AUTH="Authorization: Bearer $TOKEN"
|
||||
|
||||
echo "==> labeling/batches"
|
||||
curl -sS -H "$AUTH" "$BASE/api/v1/labeling/batches" | python3 -m json.tool | head -40
|
||||
|
||||
CID=$(curl -sS -H "$AUTH" "$BASE/api/v1/labeling/batches" | python3 -c "
|
||||
import sys,json
|
||||
items=json.load(sys.stdin).get('items',[])
|
||||
dam=[i for i in items if i.get('task')=='dam' and '0516' in str(i.get('batch',''))]
|
||||
print(dam[0]['campaign_id'] if dam else (items[0]['campaign_id'] if items else ''))
|
||||
")
|
||||
|
||||
if [[ -z "$CID" ]]; then
|
||||
echo "open campaign dam/batch_0516"
|
||||
CID=$(curl -sS -X POST -H "$AUTH" -H 'Content-Type: application/json' \
|
||||
"$BASE/api/v1/labeling/campaigns/open" \
|
||||
-d '{"project":"dms","task":"dam","batch":"batch_0516","mode":"batch_0516","location":"inbox"}' \
|
||||
| python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))")
|
||||
fi
|
||||
|
||||
echo "campaign_id=$CID"
|
||||
echo "==> bootstrap"
|
||||
curl -sS -H "$AUTH" "$BASE/api/v1/labeling/campaigns/$CID/bootstrap" | python3 -m json.tool | head -25
|
||||
echo "==> tasks"
|
||||
curl -sS -H "$AUTH" "$BASE/api/v1/labeling/campaigns/$CID/tasks?limit=3" | python3 -m json.tool | head -30
|
||||
echo "==> export"
|
||||
curl -sS -X POST -H "$AUTH" "$BASE/api/v1/labeling/campaigns/$CID/export" | python3 -m json.tool
|
||||
echo "OK"
|
||||
55
scripts/smoke_manifest_alignment.sh
Executable file
55
scripts/smoke_manifest_alignment.sh
Executable file
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
# 校验 workflow active_packs、train_versions 与 yaml_active 对齐（ML 自动化 P0）
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
python3 <<'PY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
root = Path(".")
|
||||
wf = yaml.safe_load((root / "workflow.registry.yaml").read_text(encoding="utf-8"))
|
||||
tv_path = root / "datasets/dms/manifests/train_versions.yaml"
|
||||
yaml_active = root / "datasets/dms/manifests/yaml_active"
|
||||
errors: list[str] = []
|
||||
|
||||
if not tv_path.is_file():
|
||||
errors.append(f"missing {tv_path}")
|
||||
else:
|
||||
tv = yaml.safe_load(tv_path.read_text(encoding="utf-8")) or {}
|
||||
for key, meta in tv.items():
|
||||
if key in ("schema",):
|
||||
continue
|
||||
if not isinstance(meta, dict):
|
||||
continue
|
||||
rel = meta.get("data_yaml")
|
||||
if not rel:
|
||||
continue
|
||||
p = root / "datasets/dms" / rel
|
||||
if not p.is_file():
|
||||
errors.append(f"train_versions[{key}] data_yaml not found: {p}")
|
||||
|
||||
for proj, pcfg in (wf.get("projects") or {}).items():
|
||||
for pack in pcfg.get("active_packs") or []:
|
||||
if proj == "dms":
|
||||
packs_file = root / pcfg.get("packs_registry", "datasets/dms/data_packs.yaml")
|
||||
if packs_file.is_file():
|
||||
packs = yaml.safe_load(packs_file.read_text(encoding="utf-8")) or {}
|
||||
if pack not in (packs.get("packs") or {}):
|
||||
errors.append(f"dms active_pack unknown in data_packs: {pack}")
|
||||
|
||||
if errors:
|
||||
print("MANIFEST_ALIGNMENT_FAIL")
|
||||
for e in errors:
|
||||
print(" -", e)
|
||||
sys.exit(1)
|
||||
|
||||
print("MANIFEST_ALIGNMENT_OK")
|
||||
print("train_versions keys:", len([k for k in yaml.safe_load(tv_path.read_text()) if k != "schema"]))
|
||||
print("yaml_active files:", len(list(yaml_active.glob("*.yaml"))))
|
||||
PY
|
||||
|
||||
echo "OK smoke_manifest_alignment"
|
||||
31
scripts/smoke_pending_gate.sh
Executable file
31
scripts/smoke_pending_gate.sh
Executable file
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# ML 自动化 P0：manifest 对齐 + pending 批次 stage 字段可读
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
bash scripts/smoke_manifest_alignment.sh
|
||||
|
||||
python3 <<'PY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
root = Path(".")
|
||||
wf = yaml.safe_load((root / "workflow.registry.yaml").read_text(encoding="utf-8"))
|
||||
sys.path.insert(0, str(root / "platform"))
|
||||
from as_platform.data.core import get_pending_report # noqa: E402
|
||||
|
||||
report = get_pending_report(wf)
|
||||
stages = {b.get("stage") for b in report.get("batches") or []}
|
||||
required = {"raw_pool", "out_for_labeling", "returned", "labeling_submitted"}
|
||||
missing = required - stages
|
||||
if missing:
|
||||
print("PENDING_GATE_WARN: no batches in stages", missing, "(ok if inbox empty)")
|
||||
else:
|
||||
print("PENDING_GATE_STAGES_OK", sorted(stages))
|
||||
print("PENDING_GATE_OK batches=", len(report.get("batches") or []))
|
||||
PY
|
||||
|
||||
echo "OK smoke_pending_gate"
|
||||
36
scripts/smoke_platform_api.sh
Executable file
36
scripts/smoke_platform_api.sh
Executable file
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# 平台 API 冒烟：labeling + fleet + health（需 platform :8787）
|
||||
set -euo pipefail
|
||||
BASE="${HSAP_API:-http://127.0.0.1:8787}"
|
||||
TOKEN=$(curl -sS -X POST "$BASE/api/v1/auth/dev/login" -H 'Content-Type: application/json' -d '{"name":"smoke"}' | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])")
|
||||
AUTH="Authorization: Bearer $TOKEN"
|
||||
|
||||
echo "==> health"
|
||||
curl -sS -H "$AUTH" "$BASE/api/v1/health" | python3 -m json.tool
|
||||
|
||||
echo "==> labeling/batches (count)"
|
||||
curl -sS -H "$AUTH" "$BASE/api/v1/labeling/batches" | python3 -c "import sys,json; print('batches', len(json.load(sys.stdin).get('items',[])))"
|
||||
|
||||
echo "==> fleet/map-config"
|
||||
curl -sS -H "$AUTH" "$BASE/api/v1/fleet/map-config" 2>/dev/null | python3 -m json.tool || echo "fleet disabled or 503"
|
||||
|
||||
echo "==> fleet/live"
|
||||
curl -sS -H "$AUTH" "$BASE/api/v1/fleet/live" 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print('vehicles', len(d.get('vehicles',[])))" || echo "fleet live skip"
|
||||
|
||||
CID=$(curl -sS -H "$AUTH" "$BASE/api/v1/labeling/batches" | python3 -c "
|
||||
import sys,json
|
||||
items=json.load(sys.stdin).get('items',[])
|
||||
print(items[0].get('campaign_id','') if items else '')
|
||||
")
|
||||
if [[ -n "$CID" ]]; then
|
||||
echo "==> export-jobs campaign=$CID"
|
||||
curl -sS -H "$AUTH" "$BASE/api/v1/labeling/campaigns/$CID/export-jobs" | python3 -m json.tool | head -15
|
||||
fi
|
||||
|
||||
bash "$(dirname "$0")/smoke_labeling_api.sh"
|
||||
bash "$(dirname "$0")/smoke_manifest_alignment.sh"
|
||||
echo "==> pending/gates"
|
||||
curl -sS -H "$AUTH" "$BASE/api/v1/pending/gates" | python3 -m json.tool
|
||||
echo "==> labeling/registry-profiles"
|
||||
curl -sS -H "$AUTH" "$BASE/api/v1/labeling/registry-profiles" | python3 -c "import sys,json; print('profiles', len(json.load(sys.stdin).get('profiles',[])))"
|
||||
echo "ALL_OK"
|
||||
@@ -27,7 +27,6 @@ EXCLUDE=(
|
||||
--exclude 'log/'
|
||||
--exclude 'tmp/'
|
||||
--exclude 'node_modules/'
|
||||
--exclude 'platform/web/dist/'
|
||||
)
|
||||
|
||||
echo "HSAP: $AS"
|
||||
|
||||
@@ -40,6 +40,15 @@ def main() -> None:
|
||||
sys.exit(1)
|
||||
return
|
||||
|
||||
if not redis_required:
|
||||
if ping_redis():
|
||||
print("Redis 已就绪")
|
||||
else:
|
||||
print("thread 模式：Redis 未运行，跳过等待（标注锁/Job 队列功能受限）")
|
||||
if not ok_db:
|
||||
sys.exit(1)
|
||||
return
|
||||
|
||||
for i in range(30):
|
||||
if ping_redis():
|
||||
print("Redis 已就绪")
|
||||
|
||||
Reference in New Issue
Block a user