217 lines
9.6 KiB
Bash
Executable File
217 lines
9.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/../scripts/bash/casan-paths.sh"
|
|
PROJECT_ROOT="$CASAN_APP_ROOT"
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
PASS=0; FAIL=0
|
|
pass(){ echo "PASS: $1"; PASS=$((PASS+1)); }
|
|
fail(){ echo "FAIL: $1"; FAIL=$((FAIL+1)); }
|
|
|
|
mkdir -p "$WORK/scripts" "$WORK/.specify/scripts/bash" "$WORK/docs/input"
|
|
cp "$PROJECT_ROOT/scripts/casan-step.mjs" "$WORK/scripts/casan-step.mjs"
|
|
|
|
cat > "$WORK/docs/input/okr-requirement.md" <<'EOF'
|
|
# OKR Requirement
|
|
- FR-01 Login
|
|
- FR-02 Create Objective
|
|
- FR-03 Create Key Result
|
|
- FR-04 Update Progress
|
|
- FR-05 Dashboard
|
|
- SCR-00 Login
|
|
- SCR-01 Dashboard
|
|
- SCR-02 Detail
|
|
- SCR-03 Create Objective
|
|
- SCR-04 Key Result Detail
|
|
EOF
|
|
cat > "$WORK/docs/technical_architecture.md" <<'EOF'
|
|
# Architecture
|
|
Frontend uses API client. Backend uses NestJS and Prisma.
|
|
EOF
|
|
|
|
cat > "$WORK/.specify/scripts/bash/model-router.sh" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
prompt_file="$1"
|
|
out_json="$2"
|
|
step="${CASAN_STEP_NAME:-unknown}"
|
|
case "$step" in
|
|
02-bd)
|
|
text="# Model Generated BD\n\n## Screen Layout\n- SCR-00 Login\n- SCR-01 Dashboard\n- SCR-02 Detail\n- SCR-03 Create Objective\n- SCR-04 Key Result Detail\n\n## API Boundary\nFrontend calls backend through src/lib/api.ts.\n"
|
|
;;
|
|
03-spec)
|
|
text="# Model Generated Spec\n\n## Requirements\n- FR-01 Login\n- FR-02 Create Objective\n- FR-03 Create Key Result\n- FR-04 Update Progress\n- FR-05 Dashboard\n\n## Acceptance Criteria\nEmployee isolation and manager visibility are testable.\n"
|
|
;;
|
|
05-plan)
|
|
text="# Model Generated Plan\n\n## Backend Modules\nauth, users, objectives, key-results.\n\n## Tests\nBackend service tests and e2e tests.\n\n## Golden regression test\nCompare manager objective list with golden fixture.\n\n## Rollback strategy\nUse rollback-manager.sh checkpoints before overwrite.\n\n## Build\nRun npm test and npm run build.\n"
|
|
;;
|
|
07-dd)
|
|
text="# Model Generated DD\n\n## Backend Design\nControllers call services and PrismaService.\n\n## Authorization\nEmployees only access owned objectives; managers/admins read all.\n\n## Progress Calculation\nProgress updates write ProgressUpdate and recalculate objective status.\n"
|
|
;;
|
|
08-testkit)
|
|
text="# Model Generated Test Cases\n\n- TC-01 login rejects wrong password.\n- TC-02 employee list returns only own objectives.\n- TC-03 manager list returns all seeded objectives.\n- TC-04 invalid objective payload returns 400.\n- TC-05 progress patch updates a key result.\n- TC-06 golden manager objective response fails on drift.\n"
|
|
;;
|
|
09-tasks)
|
|
text="# Model Generated Tasks\n\n- [X] Backend auth module with JWT and bcrypt.\n- [X] Frontend dashboard and detail pages.\n- [X] Backend and frontend tests.\n- [X] Golden regression evidence.\n"
|
|
;;
|
|
10-implement)
|
|
text="# Model Generated Implementation Draft\n\n## Backend Source\n- apps/okr/backend/src/auth/auth.service.ts\n- apps/okr/backend/src/objectives/objectives.service.ts\n\n## Frontend Source\n- apps/okr/frontend/src/lib/api.ts\n- apps/okr/frontend/src/pages/DashboardPage.tsx\n\n## Acceptance Gate\nThis draft is not accepted until STEP12 test PASS.\n"
|
|
;;
|
|
*)
|
|
text="# Model Generated SRS\n\n## Functional Requirements\n- FR-01 Login\n- FR-02 Create Objective\n- FR-03 Create Key Result\n- FR-04 Update Progress\n- FR-05 Dashboard\n\n## Non Functional Requirements\nAuthentication required.\n"
|
|
;;
|
|
esac
|
|
python3 - "$out_json" "$text" <<'PY'
|
|
import json, sys
|
|
out, text = sys.argv[1], sys.argv[2]
|
|
open(out, "w", encoding="utf-8").write(json.dumps({
|
|
"timestamp": "2026-07-09T00:00:00Z",
|
|
"text": text,
|
|
"model_id": "mock:sourcegen",
|
|
"role": "generate",
|
|
"route": "mock:primary",
|
|
"input_tokens": 21,
|
|
"output_tokens": 34,
|
|
"total_tokens": 55,
|
|
"latency_ms": 1,
|
|
"temperature": 0.2
|
|
}) + "\n")
|
|
PY
|
|
EOF
|
|
chmod +x "$WORK/.specify/scripts/bash/model-router.sh"
|
|
|
|
cat > "$WORK/.specify/scripts/bash/artifact-scan.sh" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 0
|
|
EOF
|
|
chmod +x "$WORK/.specify/scripts/bash/artifact-scan.sh"
|
|
|
|
(
|
|
cd "$WORK" || exit 1
|
|
CASAN_OUTPUT="$WORK/out-template.md" node scripts/casan-step.mjs 01-srs 1 >/dev/null
|
|
)
|
|
if grep -q "FR-05 Dashboard" "$WORK/docs/output/ipa-docs/srs/srs-mod01-okr-management.md" \
|
|
&& grep -q "source=template" "$WORK/docs/output/output_logs/001-okr-web-app/reports/01-srs-report.md"; then
|
|
pass "source-gen default mode keeps deterministic template"
|
|
else
|
|
fail "default template generation changed"
|
|
fi
|
|
|
|
(
|
|
cd "$WORK" || exit 1
|
|
CASAN_GEN_MODE=model CASAN_OUTPUT="$WORK/out-model.md" node scripts/casan-step.mjs 02-bd 1 >/dev/null
|
|
)
|
|
if grep -q "Model Generated BD" "$WORK/docs/output/ipa-docs/bd/bd-mod01-okr-management.md" \
|
|
&& grep -q "source=model" "$WORK/docs/output/output_logs/001-okr-web-app/reports/02-bd-report.md"; then
|
|
pass "source-gen model mode accepts scanned valid model output"
|
|
else
|
|
fail "model generated BD was not accepted"
|
|
fi
|
|
|
|
(
|
|
cd "$WORK" || exit 1
|
|
CASAN_GEN_MODE=model CASAN_OUTPUT="$WORK/out-spec.md" node scripts/casan-step.mjs 03-spec 1 >/dev/null
|
|
)
|
|
if grep -q "Model Generated Spec" "$WORK/docs/output/specs/001-okr-web-app/spec.md" \
|
|
&& grep -q "source=model" "$WORK/docs/output/output_logs/001-okr-web-app/reports/03-spec-report.md"; then
|
|
pass "source-gen wave B accepts model generated spec"
|
|
else
|
|
fail "model generated spec was not accepted"
|
|
fi
|
|
|
|
(
|
|
cd "$WORK" || exit 1
|
|
CASAN_GEN_MODE=model CASAN_OUTPUT="$WORK/out-plan1.md" node scripts/casan-step.mjs 05-plan 1 >/dev/null
|
|
CASAN_OUTPUT="$WORK/out-reviewplan1.md" node scripts/casan-step.mjs 06-reviewplan 1 >/dev/null
|
|
CASAN_GEN_MODE=model CASAN_OUTPUT="$WORK/out-plan2.md" node scripts/casan-step.mjs 05-plan 2 >/dev/null
|
|
CASAN_OUTPUT="$WORK/out-reviewplan2.md" node scripts/casan-step.mjs 06-reviewplan 2 >/dev/null
|
|
)
|
|
if grep -q "verdict: REJECTED" "$WORK/out-reviewplan1.md" \
|
|
&& grep -q "verdict: APPROVED" "$WORK/out-reviewplan2.md" \
|
|
&& grep -q "source=model" "$WORK/docs/output/output_logs/001-okr-web-app/reports/05-plan-report-attempt-2.md"; then
|
|
pass "source-gen wave B preserves review-plan reject then retry loop"
|
|
else
|
|
fail "plan review loop did not reject then approve"
|
|
fi
|
|
|
|
(
|
|
cd "$WORK" || exit 1
|
|
CASAN_GEN_MODE=model CASAN_OUTPUT="$WORK/out-dd.md" node scripts/casan-step.mjs 07-dd 1 >/dev/null
|
|
CASAN_GEN_MODE=model CASAN_OUTPUT="$WORK/out-testkit.md" node scripts/casan-step.mjs 08-testkit 1 >/dev/null
|
|
CASAN_GEN_MODE=model CASAN_OUTPUT="$WORK/out-tasks.md" node scripts/casan-step.mjs 09-tasks 1 >/dev/null
|
|
)
|
|
if grep -q "Model Generated DD" "$WORK/docs/output/ipa-docs/dd/dd-mod01-okr-management.md" \
|
|
&& grep -q "Model Generated Test Cases" "$WORK/docs/output/ipa-docs/testcase/testcase-mod01-okr-management.md" \
|
|
&& grep -q "Model Generated Tasks" "$WORK/docs/output/specs/001-okr-web-app/tasks.md"; then
|
|
pass "source-gen wave C accepts model generated dd testkit and tasks"
|
|
else
|
|
fail "wave C model artifacts were not accepted"
|
|
fi
|
|
|
|
(
|
|
cd "$WORK" || exit 1
|
|
CASAN_GEN_MODE=model CASAN_OUTPUT="$WORK/out-impl.md" node scripts/casan-step.mjs 10-implement 1 >/dev/null
|
|
)
|
|
if grep -q "Model Generated Implementation Draft" "$WORK/docs/output/specs/001-okr-web-app/implementation.md" \
|
|
&& [[ ! -f "$WORK/docs/output/specs/001-okr-web-app/implementation.accepted.json" ]]; then
|
|
pass "source-gen wave D creates implementation draft without accepting before tests"
|
|
else
|
|
fail "implementation draft acceptance gate was bypassed"
|
|
fi
|
|
|
|
(
|
|
cd "$WORK" || exit 1
|
|
CASAN_SOURCEGEN_TEST_CMD='printf "unit tests pass\n"' CASAN_OUTPUT="$WORK/out-runtests-pass.md" node scripts/casan-step.mjs 12-runtests 1 >/dev/null
|
|
)
|
|
if grep -q "verdict: PASS" "$WORK/out-runtests-pass.md" \
|
|
&& grep -q '"accepted": true' "$WORK/docs/output/specs/001-okr-web-app/implementation.accepted.json"; then
|
|
pass "source-gen wave D accepts implementation only after STEP12 test pass"
|
|
else
|
|
fail "STEP12 did not accept passing implementation"
|
|
fi
|
|
|
|
(
|
|
cd "$WORK" || exit 1
|
|
CASAN_SOURCEGEN_TEST_CMD='exit 9' CASAN_OUTPUT="$WORK/out-runtests-fail.md" node scripts/casan-step.mjs 12-runtests 1 >/dev/null
|
|
)
|
|
if grep -q "verdict: FAIL" "$WORK/out-runtests-fail.md" \
|
|
&& [[ ! -f "$WORK/docs/output/specs/001-okr-web-app/implementation.accepted.json" ]]; then
|
|
pass "source-gen wave D refuses implementation when STEP12 test fails"
|
|
else
|
|
fail "STEP12 accepted failing implementation"
|
|
fi
|
|
|
|
if grep -q '"harness":"H5-sourcegen-audit"' "$WORK/.specify/logs/audit/sourcegen.jsonl" \
|
|
&& grep -q '"prompt_sha256"' "$WORK/.specify/logs/audit/sourcegen.jsonl" \
|
|
&& grep -q '"model_id":"mock:sourcegen"' "$WORK/.specify/logs/audit/sourcegen.jsonl" \
|
|
&& grep -q '"harness":"H6-sourcegen-telemetry"' "$WORK/.specify/logs/level5/sourcegen-provider-usage.jsonl" \
|
|
&& grep -q '"total_tokens":55' "$WORK/.specify/logs/level5/sourcegen-provider-usage.jsonl"; then
|
|
pass "source-gen records prompt model and tokens into audit and telemetry"
|
|
else
|
|
fail "source-gen audit or telemetry is missing prompt/model/token evidence"
|
|
fi
|
|
|
|
cat > "$WORK/.specify/scripts/bash/artifact-scan.sh" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 2
|
|
EOF
|
|
chmod +x "$WORK/.specify/scripts/bash/artifact-scan.sh"
|
|
|
|
(
|
|
cd "$WORK" || exit 1
|
|
CASAN_GEN_MODE=model CASAN_OUTPUT="$WORK/out-fallback.md" node scripts/casan-step.mjs 01-srs 1 >/dev/null
|
|
)
|
|
if grep -q "Functional requirements extracted" "$WORK/docs/output/ipa-docs/srs/srs-mod01-okr-management.md" \
|
|
&& grep -q "source=template-fallback" "$WORK/docs/output/output_logs/001-okr-web-app/reports/01-srs-report.md"; then
|
|
pass "source-gen falls back to template when H4 artifact scan blocks model output"
|
|
else
|
|
fail "source-gen did not fallback after artifact scan block"
|
|
fi
|
|
|
|
echo ""
|
|
echo "===== SOURCEGEN TESTS: PASS=$PASS FAIL=$FAIL ====="
|
|
[[ "$FAIL" -eq 0 ]] || exit 1
|