Add CASAN CI harness gate
This commit is contained in:
@@ -0,0 +1,37 @@
|
|||||||
|
name: CASAN Harness CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
pull_request:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
harness:
|
||||||
|
runs-on: ci-runner
|
||||||
|
timeout-minutes: 45
|
||||||
|
env:
|
||||||
|
CASAN_CI_RUN_FRONTEND: "1"
|
||||||
|
CASAN_CI_RUN_INFRA_LAB: "0"
|
||||||
|
CASAN_GEN_MODE: "template"
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Install system dependencies
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if ! command -v node >/dev/null 2>&1; then
|
||||||
|
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
|
||||||
|
apt-get install -y nodejs
|
||||||
|
fi
|
||||||
|
node --version
|
||||||
|
npm --version
|
||||||
|
|
||||||
|
- name: Install npm dependencies
|
||||||
|
working-directory: AINative_OKR_CASAN5
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
- name: Run CASAN harness gate
|
||||||
|
working-directory: AINative_OKR_CASAN5
|
||||||
|
run: bash .specify/scripts/bash/ci-harness-gate.sh
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
# CASAN CI harness gate.
|
||||||
|
# Runs the reproducible harness suites in a safe order. `run-casan4` must run
|
||||||
|
# first because it rewrites `.specify/logs`.
|
||||||
|
#
|
||||||
|
# Env:
|
||||||
|
# CASAN_CI_RUN_FRONTEND=0|1 default 1
|
||||||
|
# CASAN_CI_RUN_INFRA_LAB=0|1 default 0 (Docker Compose lab is optional in CI)
|
||||||
|
# CASAN_CI_STEP_TIMEOUT_SEC default 600
|
||||||
|
# CASAN_CI_SUITE_FILTER optional regex; run matching suite names only
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||||
|
TESTS="$ROOT/.specify/tests"
|
||||||
|
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
SKIP=0
|
||||||
|
|
||||||
|
run() {
|
||||||
|
local name="$1"
|
||||||
|
shift
|
||||||
|
if [[ -n "${CASAN_CI_SUITE_FILTER:-}" && ! "$name" =~ $CASAN_CI_SUITE_FILTER ]]; then
|
||||||
|
skip "$name (filtered)"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "==> $name"
|
||||||
|
if run_with_timeout "$@"; then
|
||||||
|
echo "CI_GATE_PASS $name"
|
||||||
|
PASS=$((PASS + 1))
|
||||||
|
else
|
||||||
|
echo "CI_GATE_FAIL $name" >&2
|
||||||
|
FAIL=$((FAIL + 1))
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
skip() {
|
||||||
|
echo "CI_GATE_SKIP $1"
|
||||||
|
SKIP=$((SKIP + 1))
|
||||||
|
}
|
||||||
|
|
||||||
|
run_with_timeout() {
|
||||||
|
if ! command -v python3 >/dev/null 2>&1; then
|
||||||
|
"$@"
|
||||||
|
return $?
|
||||||
|
fi
|
||||||
|
CASAN_CI_STEP_TIMEOUT_SEC="${CASAN_CI_STEP_TIMEOUT_SEC:-600}" python3 - "$@" <<'PY'
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
timeout = int(os.environ.get("CASAN_CI_STEP_TIMEOUT_SEC", "600"))
|
||||||
|
cmd = sys.argv[1:]
|
||||||
|
try:
|
||||||
|
raise SystemExit(subprocess.run(cmd, timeout=timeout).returncode)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
print(f"CI_GATE_TIMEOUT seconds={timeout} command={' '.join(cmd)}", file=sys.stderr)
|
||||||
|
raise SystemExit(124)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
cd "$ROOT" || exit 1
|
||||||
|
|
||||||
|
run "run-casan4-harness" bash "$TESTS/run-casan4-harness-tests.sh"
|
||||||
|
run "adversarial-harness" bash "$TESTS/adversarial-harness-tests.sh"
|
||||||
|
run "phase1-track-a" bash "$TESTS/phase1-track-a-tests.sh"
|
||||||
|
run "phase2-track-c" bash "$TESTS/phase2-track-c-tests.sh"
|
||||||
|
run "phase2-sourcegen" bash "$TESTS/phase2-sourcegen-tests.sh"
|
||||||
|
run "phase3-evidence-pack" bash "$TESTS/phase3-evidence-pack-tests.sh"
|
||||||
|
run "phase3-model-router" bash "$TESTS/phase3-model-router-tests.sh"
|
||||||
|
run "phase-h5-approval" bash "$TESTS/phase-h5-approval-tests.sh"
|
||||||
|
run "phase-h5-infra" bash "$TESTS/phase-h5-infra-tests.sh"
|
||||||
|
run "phase-h6-agentops" bash "$TESTS/phase-h6-agentops-tests.sh"
|
||||||
|
run "phase-c7-incident" bash "$TESTS/phase-c7-incident-tests.sh"
|
||||||
|
run "phase-h4-multilingual" bash "$TESTS/phase-h4-multilingual-tests.sh"
|
||||||
|
run "phase-c6-sandbox" bash "$TESTS/phase-c6-sandbox-tests.sh"
|
||||||
|
run "phase-h4-split-inject" bash "$TESTS/phase-h4-split-inject-tests.sh"
|
||||||
|
run "phase10-traceability" bash "$TESTS/phase10-traceability-tests.sh"
|
||||||
|
|
||||||
|
if [[ "${CASAN_CI_RUN_FRONTEND:-1}" == "1" ]]; then
|
||||||
|
if command -v npm >/dev/null 2>&1; then
|
||||||
|
run "frontend-vitest" npm test -w frontend
|
||||||
|
else
|
||||||
|
skip "frontend-vitest (npm unavailable)"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
skip "frontend-vitest (CASAN_CI_RUN_FRONTEND=0)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${CASAN_CI_RUN_INFRA_LAB:-0}" == "1" ]]; then
|
||||||
|
if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
|
||||||
|
run "local-prod-infra-lab" bash "$TESTS/phase-prod-infra-lab-tests.sh"
|
||||||
|
else
|
||||||
|
skip "local-prod-infra-lab (docker compose unavailable)"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
skip "local-prod-infra-lab (CASAN_CI_RUN_INFRA_LAB=0)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "CI_GATE_SUMMARY PASS=$PASS FAIL=$FAIL SKIP=$SKIP"
|
||||||
|
[[ "$FAIL" -eq 0 ]] || exit 1
|
||||||
@@ -43,7 +43,7 @@
|
|||||||
| **03 Cloud patch** | 🟡 MVP done+test | `model-call.py` đã hỗ trợ `openai:<model>` và `anthropic:<model>` qua endpoint hard-pin + API key env; parser token usage và malformed payload có deterministic test. Còn live smoke với key thật + billing usage API ground truth. |
|
| **03 Cloud patch** | 🟡 MVP done+test | `model-call.py` đã hỗ trợ `openai:<model>` và `anthropic:<model>` qua endpoint hard-pin + API key env; parser token usage và malformed payload có deterministic test. Còn live smoke với key thật + billing usage API ground truth. |
|
||||||
| **02 LLM source-gen** | 🟡 đợt A MVP done+test | `01-srs` và `02-bd` hỗ trợ `CASAN_GEN_MODE=model`: gọi `model-router.sh --role generate`, scan H4 draft bằng `artifact-scan.sh`, validate token bắt buộc, fallback template nếu lỗi. Test `phase2-sourcegen` 3/0. Còn đợt B/C/D + full live pipeline smoke. |
|
| **02 LLM source-gen** | 🟡 đợt A MVP done+test | `01-srs` và `02-bd` hỗ trợ `CASAN_GEN_MODE=model`: gọi `model-router.sh --role generate`, scan H4 draft bằng `artifact-scan.sh`, validate token bắt buộc, fallback template nếu lỗi. Test `phase2-sourcegen` 3/0. Còn đợt B/C/D + full live pipeline smoke. |
|
||||||
| **04 Self-improve** | 📋 chưa bắt đầu | Khép vòng `casan improve`: đọc metrics/drift/hallucination → đề xuất vá → chạy lại gate. Phụ thuộc 02, 05. Bước 1: script đọc `metrics.jsonl` + `drift-report.json` → sinh backlog vá tự động. |
|
| **04 Self-improve** | 📋 chưa bắt đầu | Khép vòng `casan improve`: đọc metrics/drift/hallucination → đề xuất vá → chạy lại gate. Phụ thuộc 02, 05. Bước 1: script đọc `metrics.jsonl` + `drift-report.json` → sinh backlog vá tự động. |
|
||||||
| **05 CI/CD** | 📋 một phần (act_runner/deploy có) | Chuẩn hoá pipeline phát hành package `fpt-casan-sdd-harness` + chạy 12 suite trong CI (Vault+Docker service). Bước 1: `.gitea/workflows/harness-ci.yml` chạy toàn bộ suite + security-gate. |
|
| **05 CI/CD** | 🟡 CI gate MVP done | `.specify/scripts/bash/ci-harness-gate.sh` chạy các suite harness/hardening/sourcegen/traceability/frontend theo thứ tự an toàn, có timeout/filter; `.gitea/workflows/harness-ci.yml` gọi gate trên push/PR. Filtered local verify PASS=2/0. Còn: full gate xanh trên runner thật, xử lý A6 nếu còn chậm/treo, bật Docker infra lab nếu runner hỗ trợ, package/release artifact `fpt-casan-sdd-harness`. |
|
||||||
| **06 Onboard dự án 2** | 📋 chưa bắt đầu | Chứng minh reuse: cắm 1 repo khác + golden/corpus/input, đăng ký qua `verify-harness-reuse.sh`, không sửa gate. Phụ thuộc 01. |
|
| **06 Onboard dự án 2** | 📋 chưa bắt đầu | Chứng minh reuse: cắm 1 repo khác + golden/corpus/input, đăng ký qua `verify-harness-reuse.sh`, không sửa gate. Phụ thuộc 01. |
|
||||||
| **08 Context compression** | 📋 chưa bắt đầu | Nén prompt/context giảm token (H1.5). Làm SAU khi core ổn (nén thêm bề mặt rủi ro → cần quét lại). Bước 1: đo token baseline mỗi step, thử tóm tắt có kiểm chứng (H3 judge so sánh). |
|
| **08 Context compression** | 📋 chưa bắt đầu | Nén prompt/context giảm token (H1.5). Làm SAU khi core ổn (nén thêm bề mặt rủi ro → cần quét lại). Bước 1: đo token baseline mỗi step, thử tóm tắt có kiểm chứng (H3 judge so sánh). |
|
||||||
| **12 Domain Pack SDK** | 📋 chưa bắt đầu | Onboard bằng khai báo (golden/corpus/policy theo domain). Phụ thuộc 01, 06. |
|
| **12 Domain Pack SDK** | 📋 chưa bắt đầu | Onboard bằng khai báo (golden/corpus/policy theo domain). Phụ thuộc 01, 06. |
|
||||||
|
|||||||
@@ -23,7 +23,7 @@
|
|||||||
| 02 | `CASAN_PLAN_02_LLM_SOURCEGEN.md` | Noi LLM that vao sinh source thay template | Dot A MVP da co cho `01-srs`/`02-bd`; B/C/D con | Cao tiep |
|
| 02 | `CASAN_PLAN_02_LLM_SOURCEGEN.md` | Noi LLM that vao sinh source thay template | Dot A MVP da co cho `01-srs`/`02-bd`; B/C/D con | Cao tiep |
|
||||||
| 03 | `CASAN_PLAN_03_CLOUD_PATCH.md` | Patch cloud/OpenAI/Anthropic, bo stub | MVP da co + deterministic test; live smoke can key that | Cao cho live provider |
|
| 03 | `CASAN_PLAN_03_CLOUD_PATCH.md` | Patch cloud/OpenAI/Anthropic, bo stub | MVP da co + deterministic test; live smoke can key that | Cao cho live provider |
|
||||||
| 04 | `CASAN_PLAN_04_SELFIMPROVE.md` | Khep vong `casan improve` | Chua bat dau | Trung-Cao sau 02/05 |
|
| 04 | `CASAN_PLAN_04_SELFIMPROVE.md` | Khep vong `casan improve` | Chua bat dau | Trung-Cao sau 02/05 |
|
||||||
| 05 | `CASAN_PLAN_05_CICD.md` | CI/CD + release package | **File chi tiet dang thieu**; moi co mot phan runner/deploy | Cao cho release gate |
|
| 05 | `CASAN_PLAN_05_CICD.md` | CI/CD + release package | CI gate MVP + Gitea workflow da co; release package con | Cao cho runner green/release |
|
||||||
| 06 | `CASAN_PLAN_06_ONBOARD.md` | Onboard du an that thu 2 | Chua bat dau | Cao de chung minh reuse |
|
| 06 | `CASAN_PLAN_06_ONBOARD.md` | Onboard du an that thu 2 | Chua bat dau | Cao de chung minh reuse |
|
||||||
| 07 | `CASAN_PLAN_07_PRODUCTION_HARDENING.md` | H4/H5/H6 + Track C production controls | Core da xong; managed prod infra con | Con T2 infra that |
|
| 07 | `CASAN_PLAN_07_PRODUCTION_HARDENING.md` | H4/H5/H6 + Track C production controls | Core da xong; managed prod infra con | Con T2 infra that |
|
||||||
| 08 | `CASAN_PLAN_08_CONTEXT_COMPRESSION.md` | H1.5 context/token compression | Chua bat dau | Trung, lam sau core stability |
|
| 08 | `CASAN_PLAN_08_CONTEXT_COMPRESSION.md` | H1.5 context/token compression | Chua bat dau | Trung, lam sau core stability |
|
||||||
@@ -48,7 +48,7 @@
|
|||||||
| P1 | Plan-02 LLM source-gen dot B/C/D | Dot A da co; spec/plan/dd/tasks/code con template | Mo rong `generateArtifact` sang `03-spec`, `05-plan`, `07-dd`, `08-testkit`, `09-tasks`, sau do code step D voi STEP12 test gate |
|
| P1 | Plan-02 LLM source-gen dot B/C/D | Dot A da co; spec/plan/dd/tasks/code con template | Mo rong `generateArtifact` sang `03-spec`, `05-plan`, `07-dd`, `08-testkit`, `09-tasks`, sau do code step D voi STEP12 test gate |
|
||||||
| P2 | Plan-03 live cloud smoke | MVP cloud patch da test offline; chua co bang chung key/provider that tren may nay | Khi co `OPENAI_API_KEY`/`ANTHROPIC_API_KEY`, chay smoke va luu evidence token usage that |
|
| P2 | Plan-03 live cloud smoke | MVP cloud patch da test offline; chua co bang chung key/provider that tren may nay | Khi co `OPENAI_API_KEY`/`ANTHROPIC_API_KEY`, chay smoke va luu evidence token usage that |
|
||||||
| P3 | Managed prod infra T2 | Docker lab da chay, nhung chua the claim production Strong | Cau hinh enterprise IdP/JWKS, S3 Object Lock/QLDB, KMS default/HSM, dashboard TLS/OIDC, Slack/PagerDuty, billing API that |
|
| P3 | Managed prod infra T2 | Docker lab da chay, nhung chua the claim production Strong | Cau hinh enterprise IdP/JWKS, S3 Object Lock/QLDB, KMS default/HSM, dashboard TLS/OIDC, Slack/PagerDuty, billing API that |
|
||||||
| P4 | Plan-05 CI/CD release gate | Can CI chay tat ca suite va goi package | Tao/bo sung `CASAN_PLAN_05_CICD.md`; them workflow chay 218 core + optional Docker infra lab |
|
| P4 | Plan-05 CI/CD runner green + release package | CI gate script/workflow da co; chua co bang chung Gitea runner green va package artifact | Chay Gitea workflow that; sau do them package/release artifact `fpt-casan-sdd-harness` |
|
||||||
| P5 | Plan-06 onboard project 2 | Chung minh harness reuse that | Chon domain thu 2, tao app/domain data/golden/corpus, dang ky registry, chay `verify-harness-reuse.sh` |
|
| P5 | Plan-06 onboard project 2 | Chung minh harness reuse that | Chon domain thu 2, tao app/domain data/golden/corpus, dang ky registry, chay `verify-harness-reuse.sh` |
|
||||||
|
|
||||||
## Con chua lam — uu tien trung binh
|
## Con chua lam — uu tien trung binh
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
# CASAN PLAN 05 — CI/CD & Release Gate
|
||||||
|
|
||||||
|
> Status 2026-07-06: **CI gate MVP implemented + locally syntax/filter verified**.
|
||||||
|
> A reusable local/CI gate script now exists, and a Gitea Actions workflow is
|
||||||
|
> wired to run it on `main` push/PR. Package publishing/release automation is
|
||||||
|
> still planned.
|
||||||
|
|
||||||
|
## Delivered
|
||||||
|
|
||||||
|
| Capability | Where | Verification |
|
||||||
|
|---|---|---|
|
||||||
|
| One-command harness CI gate | `.specify/scripts/bash/ci-harness-gate.sh` | `bash -n`; local suites are callable in safe order |
|
||||||
|
| Safe test order | `run-casan4` runs first because it rewrites `.specify/logs`; hardening suites run after | script order |
|
||||||
|
| Per-suite timeout | `CASAN_CI_STEP_TIMEOUT_SEC` protects CI from hung model/security subprocesses | full gate attempt exposed a long A6 run; timeout wrapper added |
|
||||||
|
| Filtered local verification | `CASAN_CI_SUITE_FILTER='phase2-sourcegen|phase10-traceability'` | `CI_GATE_SUMMARY PASS=2 FAIL=0 SKIP=15` |
|
||||||
|
| Source-gen gate included | `phase2-sourcegen-tests.sh` included | sourcegen 3/0 |
|
||||||
|
| Traceability gate included | `phase10-traceability-tests.sh` included | traceability 3/0 |
|
||||||
|
| Frontend runtime tests included | `npm test -w frontend` when npm is present | frontend Vitest path |
|
||||||
|
| Optional Docker local-prod lab | `CASAN_CI_RUN_INFRA_LAB=1` runs `phase-prod-infra-lab-tests.sh` only when Docker Compose is available | skip-aware by default |
|
||||||
|
| Gitea Actions workflow | `.gitea/workflows/harness-ci.yml` | workflow calls `ci-harness-gate.sh` |
|
||||||
|
|
||||||
|
## Gate contents
|
||||||
|
|
||||||
|
Required gate:
|
||||||
|
|
||||||
|
- `run-casan4-harness-tests.sh`
|
||||||
|
- `adversarial-harness-tests.sh`
|
||||||
|
- `phase1-track-a-tests.sh`
|
||||||
|
- `phase2-track-c-tests.sh`
|
||||||
|
- `phase2-sourcegen-tests.sh`
|
||||||
|
- `phase3-evidence-pack-tests.sh`
|
||||||
|
- `phase3-model-router-tests.sh`
|
||||||
|
- `phase-h5-approval-tests.sh`
|
||||||
|
- `phase-h5-infra-tests.sh`
|
||||||
|
- `phase-h6-agentops-tests.sh`
|
||||||
|
- `phase-c7-incident-tests.sh`
|
||||||
|
- `phase-h4-multilingual-tests.sh`
|
||||||
|
- `phase-c6-sandbox-tests.sh`
|
||||||
|
- `phase-h4-split-inject-tests.sh`
|
||||||
|
- `phase10-traceability-tests.sh`
|
||||||
|
- `npm test -w frontend` when npm is available
|
||||||
|
|
||||||
|
Optional gate:
|
||||||
|
|
||||||
|
- `phase-prod-infra-lab-tests.sh` when `CASAN_CI_RUN_INFRA_LAB=1` and Docker
|
||||||
|
Compose is available.
|
||||||
|
|
||||||
|
## Remaining work
|
||||||
|
|
||||||
|
| Priority | Work | Done when |
|
||||||
|
|---|---|---|
|
||||||
|
| P1 | Run the workflow on the real Gitea runner | Gitea run shows green `CASAN Harness CI` on `main` |
|
||||||
|
| P2 | Investigate slow/hung A6 full-gate local run | `phase1-track-a-tests.sh` A6 completes under timeout consistently on Mac/runner |
|
||||||
|
| P3 | Decide Docker availability on runner | `CASAN_CI_RUN_INFRA_LAB=1` is enabled only if runner mounts Docker safely |
|
||||||
|
| P4 | Package/release automation | `npm pack` or release artifact for `fpt-casan-sdd-harness` is generated and versioned |
|
||||||
|
| P5 | Backend full `npm test` unblock | Prisma/MySQL-vs-SQLite test-infra mismatch is resolved or split into a proper MySQL service job |
|
||||||
|
| P6 | Branch protection | Merge requires green CI gate |
|
||||||
|
|
||||||
|
## Local commands
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd AINative_OKR_CASAN5
|
||||||
|
bash -n .specify/scripts/bash/ci-harness-gate.sh
|
||||||
|
CASAN_CI_RUN_FRONTEND=1 CASAN_CI_RUN_INFRA_LAB=0 bash .specify/scripts/bash/ci-harness-gate.sh
|
||||||
|
CASAN_CI_RUN_INFRA_LAB=1 bash .specify/tests/phase-prod-infra-lab-tests.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- `backend npm test` is not part of the required CI gate yet because the current
|
||||||
|
repo has a known pre-existing mismatch: `schema.prisma` is MySQL-oriented but
|
||||||
|
`backend/scripts/setup-sqlite.mjs` applies migrations to SQLite.
|
||||||
|
- The workflow installs Node 20 inside the Gitea job if the runner image does
|
||||||
|
not already provide `node`/`npm`.
|
||||||
Reference in New Issue
Block a user