From d54d7f5f4c17e1eeab8264adec3ec09922631cc4 Mon Sep 17 00:00:00 2001 From: thanhnv Date: Wed, 8 Jul 2026 00:36:06 +0900 Subject: [PATCH] =?UTF-8?q?feat(plan-01):=20Phase=202=20=E2=80=94=20reloca?= =?UTF-8?q?te=20level5/agentops=20config=20+=20init-options=20to=20package?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the harness-config files out of the mixed .specify/level5 and .specify/agentops dirs (which also hold governance/domain/state) into packages/casan-harness, leaving a per-file compat symlink at each old path. Runtime state and governance stay in place. Moved (+ .specify symlink each): level5: drift-policy.yaml kpi-schema.yaml model-fallback.yaml tool-registry.yaml harness-package.json project-registry.json provider-usage-sample.json agentops: alerts.yaml hallucination-tracking.yaml metrics.yaml rate-limits.yaml tracking.yaml top: init-options.json Stays in .specify: level5/central-governance (governance), level5/golden-runs (domain, Phase 3), agentops/alerts.log (state), traceability-map.json (domain, Phase 3). Full gate: PASS=64 FAIL=0 SKIP=3 (CASAN_CI_STEP_TIMEOUT_SEC=1200). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../.specify/agentops/alerts.yaml | 87 +----------------- .../agentops/hallucination-tracking.yaml | 77 +--------------- .../.specify/agentops/metrics.yaml | 53 +---------- .../.specify/agentops/rate-limits.yaml | 37 +------- .../.specify/agentops/tracking.yaml | 63 +------------ .../.specify/init-options.json | 12 +-- .../.specify/level5/drift-policy.yaml | 21 +---- .../.specify/level5/harness-package.json | 25 +---- .../.specify/level5/kpi-schema.yaml | 28 +----- .../.specify/level5/model-fallback.yaml | 34 +------ .../.specify/level5/project-registry.json | 27 +----- .../level5/provider-usage-sample.json | 13 +-- .../.specify/level5/tool-registry.yaml | 91 +------------------ .../packages/casan-harness/agentops/.gitkeep | 0 .../casan-harness/agentops/alerts.yaml | 86 ++++++++++++++++++ .../agentops/hallucination-tracking.yaml | 76 ++++++++++++++++ .../casan-harness/agentops/metrics.yaml | 52 +++++++++++ .../casan-harness/agentops/rate-limits.yaml | 36 ++++++++ .../casan-harness/agentops/tracking.yaml | 62 +++++++++++++ .../packages/casan-harness/init-options.json | 11 +++ .../packages/casan-harness/level5/.gitkeep | 0 .../casan-harness/level5/drift-policy.yaml | 20 ++++ .../casan-harness/level5/harness-package.json | 24 +++++ .../casan-harness/level5/kpi-schema.yaml | 27 ++++++ .../casan-harness/level5/model-fallback.yaml | 33 +++++++ .../level5/project-registry.json | 26 ++++++ .../level5/provider-usage-sample.json | 12 +++ .../casan-harness/level5/tool-registry.yaml | 90 ++++++++++++++++++ 28 files changed, 568 insertions(+), 555 deletions(-) mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/agentops/alerts.yaml mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/agentops/hallucination-tracking.yaml mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/agentops/metrics.yaml mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/agentops/rate-limits.yaml mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/agentops/tracking.yaml mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/init-options.json mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/level5/drift-policy.yaml mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/level5/harness-package.json mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/level5/kpi-schema.yaml mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/level5/model-fallback.yaml mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/level5/project-registry.json mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/level5/provider-usage-sample.json mode change 100644 => 120000 AINative_OKR_CASAN5/.specify/level5/tool-registry.yaml delete mode 100644 AINative_OKR_CASAN5/packages/casan-harness/agentops/.gitkeep create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/agentops/alerts.yaml create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/agentops/hallucination-tracking.yaml create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/agentops/metrics.yaml create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/agentops/rate-limits.yaml create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/agentops/tracking.yaml create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/init-options.json delete mode 100644 AINative_OKR_CASAN5/packages/casan-harness/level5/.gitkeep create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/level5/drift-policy.yaml create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/level5/harness-package.json create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/level5/kpi-schema.yaml create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/level5/model-fallback.yaml create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/level5/project-registry.json create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/level5/provider-usage-sample.json create mode 100644 AINative_OKR_CASAN5/packages/casan-harness/level5/tool-registry.yaml diff --git a/AINative_OKR_CASAN5/.specify/agentops/alerts.yaml b/AINative_OKR_CASAN5/.specify/agentops/alerts.yaml deleted file mode 100644 index c28210e..0000000 --- a/AINative_OKR_CASAN5/.specify/agentops/alerts.yaml +++ /dev/null @@ -1,86 +0,0 @@ -version: 1.0 -description: AgentOps Alert Rules - -alerts: - - - id: ALERT-001 - name: High Latency - condition: latency_ms > 5000 - severity: medium - action: - - log - - notify - - - id: ALERT-002 - name: High Retry - condition: retry_count > 2 - severity: medium - action: - - log - - notify - - - id: ALERT-003 - name: Failure Detected - condition: status == "failed" - severity: high - action: - - log - - notify - - escalate - - - id: ALERT-004 - name: High Cost Spike - condition: cost_estimate > 0.05 - severity: medium - action: - - log - - - id: ALERT-005 - name: Hallucination Risk - condition: output contains ("maybe" or "might be incorrect") - severity: low - action: - - log - - flag - - - id: ALERT-006 - name: Token Overuse - condition: total_tokens > 5000 - severity: medium - action: - - log - - notify - -notification: - - channels: - - - type: log - path: ".specify/agentops/alerts.log" - - - type: console - enabled: true - - # future integration - # - type: slack - # webhook: "https://hooks.slack.com/..." - -escalation: - - enabled: true - - rules: - - severity: high - action: immediate_alert - - - severity: medium - action: batched_alert - - - severity: low - action: log_only - -level4_gate: - alert_log_must_exist: true - failed_step_must_emit_alert: true - retry_threshold_must_emit_alert: true - cost_spike_must_emit_alert: true diff --git a/AINative_OKR_CASAN5/.specify/agentops/alerts.yaml b/AINative_OKR_CASAN5/.specify/agentops/alerts.yaml new file mode 120000 index 0000000..29570f8 --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/agentops/alerts.yaml @@ -0,0 +1 @@ +../../packages/casan-harness/agentops/alerts.yaml \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/agentops/hallucination-tracking.yaml b/AINative_OKR_CASAN5/.specify/agentops/hallucination-tracking.yaml deleted file mode 100644 index 684dad9..0000000 --- a/AINative_OKR_CASAN5/.specify/agentops/hallucination-tracking.yaml +++ /dev/null @@ -1,76 +0,0 @@ -version: 1.0 -description: CASAN H6 per-step hallucination signal tracking - -tracking: - enabled: true - method: cross_reference_check - log_path: .specify/logs/cost/metrics.jsonl - field_name: hallucination_signals - -steps_with_tracking: - - step: step-1-srs - reference_docs: - - docs/input/okr-requirement.md - signals: - - ungrounded_feature_claim - - invented_requirement - - undefined_acronym - keywords_indicating_hallucination: - - "I assume" - - "typically" - - "usually includes" - - "standard practice suggests" - - - step: step-3-spec - reference_docs: - - docs/output/ipa-docs/srs/ - - docs/output/ipa-docs/bd/ - signals: - - invented_api_endpoint - - schema_mismatch - - undefined_term - - ungrounded_business_rule - keywords_indicating_hallucination: - - "assumed from context" - - "common pattern would be" - - "likely the intent is" - - - step: step-6-plan - reference_docs: - - docs/output/specs/ - signals: - - invented_dependency - - wrong_module_assignment - - unspecified_library - keywords_indicating_hallucination: - - "might need" - - "could potentially use" - - "I believe" - - - step: step-10-implement - reference_docs: - - docs/output/specs/ - signals: - - undocumented_endpoint - - wrong_field_type - - missing_required_field - - hardcoded_value_not_in_spec - keywords_indicating_hallucination: - - "// TODO: confirm" - - "// assumed" - - "// placeholder" - - - step: step-11-review-code - reference_docs: - - docs/output/specs/ - - backend/ - - frontend/ - signals: - - false_security_clearance - - invented_test_result - - unverified_compliance_claim - -thresholds: - warn: 3 - block: 7 - note: "warn=log warning; block=force retry of step" diff --git a/AINative_OKR_CASAN5/.specify/agentops/hallucination-tracking.yaml b/AINative_OKR_CASAN5/.specify/agentops/hallucination-tracking.yaml new file mode 120000 index 0000000..ce38990 --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/agentops/hallucination-tracking.yaml @@ -0,0 +1 @@ +../../packages/casan-harness/agentops/hallucination-tracking.yaml \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/agentops/metrics.yaml b/AINative_OKR_CASAN5/.specify/agentops/metrics.yaml deleted file mode 100644 index 63daf37..0000000 --- a/AINative_OKR_CASAN5/.specify/agentops/metrics.yaml +++ /dev/null @@ -1,52 +0,0 @@ -version: 1.0 -description: CASAN H6 AgentOps metrics and scoring configuration - -collection: - enabled: true - trace_format: json - metrics_format: jsonl - trace_dir: ".specify/logs/trace" - metrics_log: ".specify/logs/cost/metrics.jsonl" - alert_log: ".specify/agentops/alerts.log" - -dimensions: - - trace_id - - timestamp - - agent - - step - - status - - exit_code - - latency_ms - - retry_count - - input_tokens - - output_tokens - - total_tokens - - cost_estimate - - alerts - - input_hash - - output_hash - -cost_model: - default_cost_per_1k_tokens_usd: 0.002 - override_env: CASAN_COST_PER_1K - -thresholds: - latency_alert_ms: 5000 - retry_alert_threshold: 2 - token_alert_threshold: 5000 - failure_alert: true - -level4_gate: - required: true - minimum_evidence: - traces_per_pipeline_step: 1 - metrics_log_present: true - alert_rules_present: true - failure_status_recorded: true - -level5_readiness: - required_additions: - - drift_detection_against_golden_runs - - model_fallback_routing - - live_dashboard - - business_outcome_metrics diff --git a/AINative_OKR_CASAN5/.specify/agentops/metrics.yaml b/AINative_OKR_CASAN5/.specify/agentops/metrics.yaml new file mode 120000 index 0000000..6cbbaee --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/agentops/metrics.yaml @@ -0,0 +1 @@ +../../packages/casan-harness/agentops/metrics.yaml \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/agentops/rate-limits.yaml b/AINative_OKR_CASAN5/.specify/agentops/rate-limits.yaml deleted file mode 100644 index 3429e90..0000000 --- a/AINative_OKR_CASAN5/.specify/agentops/rate-limits.yaml +++ /dev/null @@ -1,36 +0,0 @@ -version: 1.0 -description: CASAN H2 Tool rate limits — prevent runaway agent loops - -limits: - - tool: deploy - max_per_pipeline: 3 - window: pipeline_run - note: "Max 3 deploy attempts per pipeline run before escalation" - - - tool: migration - max_per_pipeline: 5 - window: pipeline_run - note: "Max 5 migration runs — includes retries" - - - tool: db_write - max_per_pipeline: 10 - window: pipeline_run - - - tool: write_code - max_per_pipeline: 50 - window: pipeline_run - note: "50 write_code calls covers BE+FE implementation + up to 5 retry cycles" - - - tool: external_api - max_per_hour: 100 - window: rolling_60min - - - tool: agent_step - max_per_pipeline: 200 - window: pipeline_run - note: "Covers all 13 steps × max retries" - -enforcement: - on_limit_exceeded: deny - log_denials: true - log_path: .specify/logs/audit/rate-limit-denials.jsonl diff --git a/AINative_OKR_CASAN5/.specify/agentops/rate-limits.yaml b/AINative_OKR_CASAN5/.specify/agentops/rate-limits.yaml new file mode 120000 index 0000000..2e58f8c --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/agentops/rate-limits.yaml @@ -0,0 +1 @@ +../../packages/casan-harness/agentops/rate-limits.yaml \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/agentops/tracking.yaml b/AINative_OKR_CASAN5/.specify/agentops/tracking.yaml deleted file mode 100644 index 190211b..0000000 --- a/AINative_OKR_CASAN5/.specify/agentops/tracking.yaml +++ /dev/null @@ -1,62 +0,0 @@ -version: 1.0 -description: Agent Tracking & Tracing Configuration - -tracking: - - enabled: true - - trace: - - fields: - - trace_id - - timestamp - - user_input - - processed_input - - output - - latency_ms - - tokens_input - - tokens_output - - total_tokens - - cost_estimate - - retry_count - - status - - error - - correlation: - - enable: true - key: trace_id - - stages: - - - name: input_received - log: true - - - name: security_checked - log: true - - - name: governance_checked - log: true - - - name: llm_called - log: true - - - name: output_filtered - log: true - - storage: - - type: file - path: ".specify/logs/trace/" - - format: json - - retention: - - days: 30 - -logging: - - level: info - include_input: true - include_output: true \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/agentops/tracking.yaml b/AINative_OKR_CASAN5/.specify/agentops/tracking.yaml new file mode 120000 index 0000000..a104615 --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/agentops/tracking.yaml @@ -0,0 +1 @@ +../../packages/casan-harness/agentops/tracking.yaml \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/init-options.json b/AINative_OKR_CASAN5/.specify/init-options.json deleted file mode 100644 index 78d6d45..0000000 --- a/AINative_OKR_CASAN5/.specify/init-options.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "ai": "claude", - "ai_commands_dir": ".claude/commands", - "ai_skills": false, - "branch_numbering": "sequential", - "here": true, - "offline": true, - "preset": null, - "script": "sh", - "speckit_version": "0.4.0" -} \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/init-options.json b/AINative_OKR_CASAN5/.specify/init-options.json new file mode 120000 index 0000000..10aa740 --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/init-options.json @@ -0,0 +1 @@ +../packages/casan-harness/init-options.json \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/level5/drift-policy.yaml b/AINative_OKR_CASAN5/.specify/level5/drift-policy.yaml deleted file mode 100644 index b978f76..0000000 --- a/AINative_OKR_CASAN5/.specify/level5/drift-policy.yaml +++ /dev/null @@ -1,20 +0,0 @@ -version: 1.0 -description: CASAN Level 5 drift detection policy - -golden_dir: .specify/level5/golden-runs -evidence_dir: docs/output/casan/level5-evidence - -thresholds: - minimum_similarity: 0.85 - maximum_length_delta_ratio: 0.30 - -actions: - pass: allow - warn: require_review - fail: block_or_fallback - -metrics: - - similarity_ratio - - length_delta_ratio - - golden_hash - - candidate_hash diff --git a/AINative_OKR_CASAN5/.specify/level5/drift-policy.yaml b/AINative_OKR_CASAN5/.specify/level5/drift-policy.yaml new file mode 120000 index 0000000..ea6df33 --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/level5/drift-policy.yaml @@ -0,0 +1 @@ +../../packages/casan-harness/level5/drift-policy.yaml \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/level5/harness-package.json b/AINative_OKR_CASAN5/.specify/level5/harness-package.json deleted file mode 100644 index 72ef9df..0000000 --- a/AINative_OKR_CASAN5/.specify/level5/harness-package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "package": "fpt-casan-sdd-harness", - "version": "1.0.0", - "capabilities": [ - "H4-security", - "H5-governance", - "H6-agentops", - "L5-drift-detection", - "L5-model-fallback", - "L5-tool-registry", - "L5-rollback", - "L5-business-feedback" - ], - "compatibility": { - "speckit": ">=1.0.0", - "shell": "bash", - "python": ">=3.9" - }, - "entrypoints": { - "wrapper": ".specify/scripts/bash/casan-harness.sh", - "test": ".specify/tests/run-casan4-harness-tests.sh", - "dashboard": "docs/output/casan/agentops-dashboard.html" - } -} diff --git a/AINative_OKR_CASAN5/.specify/level5/harness-package.json b/AINative_OKR_CASAN5/.specify/level5/harness-package.json new file mode 120000 index 0000000..abae93c --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/level5/harness-package.json @@ -0,0 +1 @@ +../../packages/casan-harness/level5/harness-package.json \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/level5/kpi-schema.yaml b/AINative_OKR_CASAN5/.specify/level5/kpi-schema.yaml deleted file mode 100644 index 14d8923..0000000 --- a/AINative_OKR_CASAN5/.specify/level5/kpi-schema.yaml +++ /dev/null @@ -1,27 +0,0 @@ -version: 1.0 -description: CASAN Level 5 business KPI feedback schema - -kpis: - - id: cycle_time_minutes - direction: lower_is_better - baseline: 180 - target: 90 - - - id: review_rejection_rate - direction: lower_is_better - baseline: 0.30 - target: 0.10 - - - id: defect_leakage_rate - direction: lower_is_better - baseline: 0.15 - target: 0.05 - - - id: manual_rework_hours - direction: lower_is_better - baseline: 12 - target: 4 - -evidence: - input: docs/output/casan/level5-evidence/business-kpi-input.json - output: docs/output/casan/level5-evidence/business-kpi-report.json diff --git a/AINative_OKR_CASAN5/.specify/level5/kpi-schema.yaml b/AINative_OKR_CASAN5/.specify/level5/kpi-schema.yaml new file mode 120000 index 0000000..235cecc --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/level5/kpi-schema.yaml @@ -0,0 +1 @@ +../../packages/casan-harness/level5/kpi-schema.yaml \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/level5/model-fallback.yaml b/AINative_OKR_CASAN5/.specify/level5/model-fallback.yaml deleted file mode 100644 index 6dab422..0000000 --- a/AINative_OKR_CASAN5/.specify/level5/model-fallback.yaml +++ /dev/null @@ -1,33 +0,0 @@ -version: 1.0 -description: CASAN Level 5 model/tool fallback policy - -routes: - - step_pattern: "review|test|security" - primary: primary-model - fallback: verifier-model - max_attempts: 2 - fallback_on: - - nonzero_exit - - empty_output - - timeout - - - step_pattern: "implement|write_code|deploy" - primary: primary-model - fallback: conservative-model - max_attempts: 1 - fallback_on: - - nonzero_exit - - policy_reject - governance_required: true - -default_route: - primary: primary-model - fallback: fallback-model - max_attempts: 2 - fallback_on: - - nonzero_exit - - empty_output - -evidence: - trace_dir: .specify/logs/trace - fallback_log: .specify/logs/level5/fallback.jsonl diff --git a/AINative_OKR_CASAN5/.specify/level5/model-fallback.yaml b/AINative_OKR_CASAN5/.specify/level5/model-fallback.yaml new file mode 120000 index 0000000..8099659 --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/level5/model-fallback.yaml @@ -0,0 +1 @@ +../../packages/casan-harness/level5/model-fallback.yaml \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/level5/project-registry.json b/AINative_OKR_CASAN5/.specify/level5/project-registry.json deleted file mode 100644 index e115755..0000000 --- a/AINative_OKR_CASAN5/.specify/level5/project-registry.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "registry": "fpt-casan-harness-adoption", - "projects": [ - { - "project_id": "AINative_OKR_CASAN4", - "domain": "SDD OKR", - "harness_package": "fpt-casan-sdd-harness", - "harness_version": "1.0.0", - "status": "active" - }, - { - "project_id": "CASAN_DEMO_PROJECT_A", - "domain": "Demo SDLC", - "harness_package": "fpt-casan-sdd-harness", - "harness_version": "1.0.0", - "status": "registered" - }, - { - "project_id": "CASAN_DEMO_PROJECT_B", - "domain": "Demo QA", - "harness_package": "fpt-casan-sdd-harness", - "harness_version": "1.0.0", - "status": "registered" - } - ] -} diff --git a/AINative_OKR_CASAN5/.specify/level5/project-registry.json b/AINative_OKR_CASAN5/.specify/level5/project-registry.json new file mode 120000 index 0000000..046b280 --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/level5/project-registry.json @@ -0,0 +1 @@ +../../packages/casan-harness/level5/project-registry.json \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/level5/provider-usage-sample.json b/AINative_OKR_CASAN5/.specify/level5/provider-usage-sample.json deleted file mode 100644 index 9e420b7..0000000 --- a/AINative_OKR_CASAN5/.specify/level5/provider-usage-sample.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "provider": "sample-provider", - "model": "sample-model-large", - "run_id": "provider-run-001", - "step": "speckit.implement", - "input_tokens": 1842, - "output_tokens": 936, - "total_tokens": 2778, - "cost_usd": 0.08334, - "latency_ms": 4210, - "status": "success" -} diff --git a/AINative_OKR_CASAN5/.specify/level5/provider-usage-sample.json b/AINative_OKR_CASAN5/.specify/level5/provider-usage-sample.json new file mode 120000 index 0000000..8d6bf5a --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/level5/provider-usage-sample.json @@ -0,0 +1 @@ +../../packages/casan-harness/level5/provider-usage-sample.json \ No newline at end of file diff --git a/AINative_OKR_CASAN5/.specify/level5/tool-registry.yaml b/AINative_OKR_CASAN5/.specify/level5/tool-registry.yaml deleted file mode 100644 index def84ae..0000000 --- a/AINative_OKR_CASAN5/.specify/level5/tool-registry.yaml +++ /dev/null @@ -1,90 +0,0 @@ -version: 1.0 -description: CASAN Level 5 tool registry with ownership, schema, risk, idempotency, and rollback metadata - -tools: - - id: agent_step - owner: ai-platform - risk_level: low - side_effect: false - idempotency_required: false - timeout_seconds: 600 - - - id: write_code - owner: engineering - risk_level: high - side_effect: true - idempotency_required: true - timeout_seconds: 900 - allowed_agents: "boss,implement-agent" - rollback: - strategy: git_patch_reverse - script: .specify/scripts/bash/rollback-manager.sh - - - id: migration - owner: database-owner - risk_level: high - side_effect: true - idempotency_required: true - timeout_seconds: 900 - allowed_agents: "boss,implement-agent" - rollback: - strategy: migration_down_or_restore - script: .specify/scripts/bash/rollback-manager.sh - - - id: deploy - owner: release-manager - risk_level: high - side_effect: true - idempotency_required: true - timeout_seconds: 1200 - rate_limit_per_run: 2 - allowed_agents: "boss,release-manager" - rollback: - strategy: previous_release - script: .specify/scripts/bash/rollback-manager.sh - - - id: external_api - owner: platform-team - risk_level: medium - side_effect: true - idempotency_required: true - timeout_seconds: 120 - allowed_agents: "boss,implement-agent" - rollback: - strategy: compensating_request - script: .specify/scripts/bash/rollback-manager.sh - - - id: db_write - owner: database-owner - risk_level: high - side_effect: true - idempotency_required: true - timeout_seconds: 300 - allowed_agents: "boss,implement-agent" - rollback: - strategy: restore_from_backup - script: .specify/scripts/powershell/rollback-manager.ps1 - - - id: write_file - owner: engineering - risk_level: medium - side_effect: true - idempotency_required: false - timeout_seconds: 60 - allowed_agents: "boss,implement-agent,design-agent" - rollback: - strategy: delete_or_restore - script: .specify/scripts/bash/rollback-manager.sh - -level5_gate: - required_fields: - - id - - owner - - risk_level - - side_effect - - idempotency_required - - timeout_seconds - side_effecting_tools_require: - - idempotency_key - - audit_record - - rollback_strategy diff --git a/AINative_OKR_CASAN5/.specify/level5/tool-registry.yaml b/AINative_OKR_CASAN5/.specify/level5/tool-registry.yaml new file mode 120000 index 0000000..fe54e0c --- /dev/null +++ b/AINative_OKR_CASAN5/.specify/level5/tool-registry.yaml @@ -0,0 +1 @@ +../../packages/casan-harness/level5/tool-registry.yaml \ No newline at end of file diff --git a/AINative_OKR_CASAN5/packages/casan-harness/agentops/.gitkeep b/AINative_OKR_CASAN5/packages/casan-harness/agentops/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/AINative_OKR_CASAN5/packages/casan-harness/agentops/alerts.yaml b/AINative_OKR_CASAN5/packages/casan-harness/agentops/alerts.yaml new file mode 100644 index 0000000..c28210e --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/agentops/alerts.yaml @@ -0,0 +1,86 @@ +version: 1.0 +description: AgentOps Alert Rules + +alerts: + + - id: ALERT-001 + name: High Latency + condition: latency_ms > 5000 + severity: medium + action: + - log + - notify + + - id: ALERT-002 + name: High Retry + condition: retry_count > 2 + severity: medium + action: + - log + - notify + + - id: ALERT-003 + name: Failure Detected + condition: status == "failed" + severity: high + action: + - log + - notify + - escalate + + - id: ALERT-004 + name: High Cost Spike + condition: cost_estimate > 0.05 + severity: medium + action: + - log + + - id: ALERT-005 + name: Hallucination Risk + condition: output contains ("maybe" or "might be incorrect") + severity: low + action: + - log + - flag + + - id: ALERT-006 + name: Token Overuse + condition: total_tokens > 5000 + severity: medium + action: + - log + - notify + +notification: + + channels: + + - type: log + path: ".specify/agentops/alerts.log" + + - type: console + enabled: true + + # future integration + # - type: slack + # webhook: "https://hooks.slack.com/..." + +escalation: + + enabled: true + + rules: + - severity: high + action: immediate_alert + + - severity: medium + action: batched_alert + + - severity: low + action: log_only + +level4_gate: + alert_log_must_exist: true + failed_step_must_emit_alert: true + retry_threshold_must_emit_alert: true + cost_spike_must_emit_alert: true diff --git a/AINative_OKR_CASAN5/packages/casan-harness/agentops/hallucination-tracking.yaml b/AINative_OKR_CASAN5/packages/casan-harness/agentops/hallucination-tracking.yaml new file mode 100644 index 0000000..684dad9 --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/agentops/hallucination-tracking.yaml @@ -0,0 +1,76 @@ +version: 1.0 +description: CASAN H6 per-step hallucination signal tracking + +tracking: + enabled: true + method: cross_reference_check + log_path: .specify/logs/cost/metrics.jsonl + field_name: hallucination_signals + +steps_with_tracking: + - step: step-1-srs + reference_docs: + - docs/input/okr-requirement.md + signals: + - ungrounded_feature_claim + - invented_requirement + - undefined_acronym + keywords_indicating_hallucination: + - "I assume" + - "typically" + - "usually includes" + - "standard practice suggests" + + - step: step-3-spec + reference_docs: + - docs/output/ipa-docs/srs/ + - docs/output/ipa-docs/bd/ + signals: + - invented_api_endpoint + - schema_mismatch + - undefined_term + - ungrounded_business_rule + keywords_indicating_hallucination: + - "assumed from context" + - "common pattern would be" + - "likely the intent is" + + - step: step-6-plan + reference_docs: + - docs/output/specs/ + signals: + - invented_dependency + - wrong_module_assignment + - unspecified_library + keywords_indicating_hallucination: + - "might need" + - "could potentially use" + - "I believe" + + - step: step-10-implement + reference_docs: + - docs/output/specs/ + signals: + - undocumented_endpoint + - wrong_field_type + - missing_required_field + - hardcoded_value_not_in_spec + keywords_indicating_hallucination: + - "// TODO: confirm" + - "// assumed" + - "// placeholder" + + - step: step-11-review-code + reference_docs: + - docs/output/specs/ + - backend/ + - frontend/ + signals: + - false_security_clearance + - invented_test_result + - unverified_compliance_claim + +thresholds: + warn: 3 + block: 7 + note: "warn=log warning; block=force retry of step" diff --git a/AINative_OKR_CASAN5/packages/casan-harness/agentops/metrics.yaml b/AINative_OKR_CASAN5/packages/casan-harness/agentops/metrics.yaml new file mode 100644 index 0000000..63daf37 --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/agentops/metrics.yaml @@ -0,0 +1,52 @@ +version: 1.0 +description: CASAN H6 AgentOps metrics and scoring configuration + +collection: + enabled: true + trace_format: json + metrics_format: jsonl + trace_dir: ".specify/logs/trace" + metrics_log: ".specify/logs/cost/metrics.jsonl" + alert_log: ".specify/agentops/alerts.log" + +dimensions: + - trace_id + - timestamp + - agent + - step + - status + - exit_code + - latency_ms + - retry_count + - input_tokens + - output_tokens + - total_tokens + - cost_estimate + - alerts + - input_hash + - output_hash + +cost_model: + default_cost_per_1k_tokens_usd: 0.002 + override_env: CASAN_COST_PER_1K + +thresholds: + latency_alert_ms: 5000 + retry_alert_threshold: 2 + token_alert_threshold: 5000 + failure_alert: true + +level4_gate: + required: true + minimum_evidence: + traces_per_pipeline_step: 1 + metrics_log_present: true + alert_rules_present: true + failure_status_recorded: true + +level5_readiness: + required_additions: + - drift_detection_against_golden_runs + - model_fallback_routing + - live_dashboard + - business_outcome_metrics diff --git a/AINative_OKR_CASAN5/packages/casan-harness/agentops/rate-limits.yaml b/AINative_OKR_CASAN5/packages/casan-harness/agentops/rate-limits.yaml new file mode 100644 index 0000000..3429e90 --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/agentops/rate-limits.yaml @@ -0,0 +1,36 @@ +version: 1.0 +description: CASAN H2 Tool rate limits — prevent runaway agent loops + +limits: + - tool: deploy + max_per_pipeline: 3 + window: pipeline_run + note: "Max 3 deploy attempts per pipeline run before escalation" + + - tool: migration + max_per_pipeline: 5 + window: pipeline_run + note: "Max 5 migration runs — includes retries" + + - tool: db_write + max_per_pipeline: 10 + window: pipeline_run + + - tool: write_code + max_per_pipeline: 50 + window: pipeline_run + note: "50 write_code calls covers BE+FE implementation + up to 5 retry cycles" + + - tool: external_api + max_per_hour: 100 + window: rolling_60min + + - tool: agent_step + max_per_pipeline: 200 + window: pipeline_run + note: "Covers all 13 steps × max retries" + +enforcement: + on_limit_exceeded: deny + log_denials: true + log_path: .specify/logs/audit/rate-limit-denials.jsonl diff --git a/AINative_OKR_CASAN5/packages/casan-harness/agentops/tracking.yaml b/AINative_OKR_CASAN5/packages/casan-harness/agentops/tracking.yaml new file mode 100644 index 0000000..190211b --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/agentops/tracking.yaml @@ -0,0 +1,62 @@ +version: 1.0 +description: Agent Tracking & Tracing Configuration + +tracking: + + enabled: true + + trace: + + fields: + - trace_id + - timestamp + - user_input + - processed_input + - output + - latency_ms + - tokens_input + - tokens_output + - total_tokens + - cost_estimate + - retry_count + - status + - error + + correlation: + + enable: true + key: trace_id + + stages: + + - name: input_received + log: true + + - name: security_checked + log: true + + - name: governance_checked + log: true + + - name: llm_called + log: true + + - name: output_filtered + log: true + + storage: + + type: file + path: ".specify/logs/trace/" + + format: json + + retention: + + days: 30 + +logging: + + level: info + include_input: true + include_output: true \ No newline at end of file diff --git a/AINative_OKR_CASAN5/packages/casan-harness/init-options.json b/AINative_OKR_CASAN5/packages/casan-harness/init-options.json new file mode 100644 index 0000000..78d6d45 --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/init-options.json @@ -0,0 +1,11 @@ +{ + "ai": "claude", + "ai_commands_dir": ".claude/commands", + "ai_skills": false, + "branch_numbering": "sequential", + "here": true, + "offline": true, + "preset": null, + "script": "sh", + "speckit_version": "0.4.0" +} \ No newline at end of file diff --git a/AINative_OKR_CASAN5/packages/casan-harness/level5/.gitkeep b/AINative_OKR_CASAN5/packages/casan-harness/level5/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/AINative_OKR_CASAN5/packages/casan-harness/level5/drift-policy.yaml b/AINative_OKR_CASAN5/packages/casan-harness/level5/drift-policy.yaml new file mode 100644 index 0000000..b978f76 --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/level5/drift-policy.yaml @@ -0,0 +1,20 @@ +version: 1.0 +description: CASAN Level 5 drift detection policy + +golden_dir: .specify/level5/golden-runs +evidence_dir: docs/output/casan/level5-evidence + +thresholds: + minimum_similarity: 0.85 + maximum_length_delta_ratio: 0.30 + +actions: + pass: allow + warn: require_review + fail: block_or_fallback + +metrics: + - similarity_ratio + - length_delta_ratio + - golden_hash + - candidate_hash diff --git a/AINative_OKR_CASAN5/packages/casan-harness/level5/harness-package.json b/AINative_OKR_CASAN5/packages/casan-harness/level5/harness-package.json new file mode 100644 index 0000000..72ef9df --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/level5/harness-package.json @@ -0,0 +1,24 @@ +{ + "package": "fpt-casan-sdd-harness", + "version": "1.0.0", + "capabilities": [ + "H4-security", + "H5-governance", + "H6-agentops", + "L5-drift-detection", + "L5-model-fallback", + "L5-tool-registry", + "L5-rollback", + "L5-business-feedback" + ], + "compatibility": { + "speckit": ">=1.0.0", + "shell": "bash", + "python": ">=3.9" + }, + "entrypoints": { + "wrapper": ".specify/scripts/bash/casan-harness.sh", + "test": ".specify/tests/run-casan4-harness-tests.sh", + "dashboard": "docs/output/casan/agentops-dashboard.html" + } +} diff --git a/AINative_OKR_CASAN5/packages/casan-harness/level5/kpi-schema.yaml b/AINative_OKR_CASAN5/packages/casan-harness/level5/kpi-schema.yaml new file mode 100644 index 0000000..14d8923 --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/level5/kpi-schema.yaml @@ -0,0 +1,27 @@ +version: 1.0 +description: CASAN Level 5 business KPI feedback schema + +kpis: + - id: cycle_time_minutes + direction: lower_is_better + baseline: 180 + target: 90 + + - id: review_rejection_rate + direction: lower_is_better + baseline: 0.30 + target: 0.10 + + - id: defect_leakage_rate + direction: lower_is_better + baseline: 0.15 + target: 0.05 + + - id: manual_rework_hours + direction: lower_is_better + baseline: 12 + target: 4 + +evidence: + input: docs/output/casan/level5-evidence/business-kpi-input.json + output: docs/output/casan/level5-evidence/business-kpi-report.json diff --git a/AINative_OKR_CASAN5/packages/casan-harness/level5/model-fallback.yaml b/AINative_OKR_CASAN5/packages/casan-harness/level5/model-fallback.yaml new file mode 100644 index 0000000..6dab422 --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/level5/model-fallback.yaml @@ -0,0 +1,33 @@ +version: 1.0 +description: CASAN Level 5 model/tool fallback policy + +routes: + - step_pattern: "review|test|security" + primary: primary-model + fallback: verifier-model + max_attempts: 2 + fallback_on: + - nonzero_exit + - empty_output + - timeout + + - step_pattern: "implement|write_code|deploy" + primary: primary-model + fallback: conservative-model + max_attempts: 1 + fallback_on: + - nonzero_exit + - policy_reject + governance_required: true + +default_route: + primary: primary-model + fallback: fallback-model + max_attempts: 2 + fallback_on: + - nonzero_exit + - empty_output + +evidence: + trace_dir: .specify/logs/trace + fallback_log: .specify/logs/level5/fallback.jsonl diff --git a/AINative_OKR_CASAN5/packages/casan-harness/level5/project-registry.json b/AINative_OKR_CASAN5/packages/casan-harness/level5/project-registry.json new file mode 100644 index 0000000..e115755 --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/level5/project-registry.json @@ -0,0 +1,26 @@ +{ + "registry": "fpt-casan-harness-adoption", + "projects": [ + { + "project_id": "AINative_OKR_CASAN4", + "domain": "SDD OKR", + "harness_package": "fpt-casan-sdd-harness", + "harness_version": "1.0.0", + "status": "active" + }, + { + "project_id": "CASAN_DEMO_PROJECT_A", + "domain": "Demo SDLC", + "harness_package": "fpt-casan-sdd-harness", + "harness_version": "1.0.0", + "status": "registered" + }, + { + "project_id": "CASAN_DEMO_PROJECT_B", + "domain": "Demo QA", + "harness_package": "fpt-casan-sdd-harness", + "harness_version": "1.0.0", + "status": "registered" + } + ] +} diff --git a/AINative_OKR_CASAN5/packages/casan-harness/level5/provider-usage-sample.json b/AINative_OKR_CASAN5/packages/casan-harness/level5/provider-usage-sample.json new file mode 100644 index 0000000..9e420b7 --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/level5/provider-usage-sample.json @@ -0,0 +1,12 @@ +{ + "provider": "sample-provider", + "model": "sample-model-large", + "run_id": "provider-run-001", + "step": "speckit.implement", + "input_tokens": 1842, + "output_tokens": 936, + "total_tokens": 2778, + "cost_usd": 0.08334, + "latency_ms": 4210, + "status": "success" +} diff --git a/AINative_OKR_CASAN5/packages/casan-harness/level5/tool-registry.yaml b/AINative_OKR_CASAN5/packages/casan-harness/level5/tool-registry.yaml new file mode 100644 index 0000000..def84ae --- /dev/null +++ b/AINative_OKR_CASAN5/packages/casan-harness/level5/tool-registry.yaml @@ -0,0 +1,90 @@ +version: 1.0 +description: CASAN Level 5 tool registry with ownership, schema, risk, idempotency, and rollback metadata + +tools: + - id: agent_step + owner: ai-platform + risk_level: low + side_effect: false + idempotency_required: false + timeout_seconds: 600 + + - id: write_code + owner: engineering + risk_level: high + side_effect: true + idempotency_required: true + timeout_seconds: 900 + allowed_agents: "boss,implement-agent" + rollback: + strategy: git_patch_reverse + script: .specify/scripts/bash/rollback-manager.sh + + - id: migration + owner: database-owner + risk_level: high + side_effect: true + idempotency_required: true + timeout_seconds: 900 + allowed_agents: "boss,implement-agent" + rollback: + strategy: migration_down_or_restore + script: .specify/scripts/bash/rollback-manager.sh + + - id: deploy + owner: release-manager + risk_level: high + side_effect: true + idempotency_required: true + timeout_seconds: 1200 + rate_limit_per_run: 2 + allowed_agents: "boss,release-manager" + rollback: + strategy: previous_release + script: .specify/scripts/bash/rollback-manager.sh + + - id: external_api + owner: platform-team + risk_level: medium + side_effect: true + idempotency_required: true + timeout_seconds: 120 + allowed_agents: "boss,implement-agent" + rollback: + strategy: compensating_request + script: .specify/scripts/bash/rollback-manager.sh + + - id: db_write + owner: database-owner + risk_level: high + side_effect: true + idempotency_required: true + timeout_seconds: 300 + allowed_agents: "boss,implement-agent" + rollback: + strategy: restore_from_backup + script: .specify/scripts/powershell/rollback-manager.ps1 + + - id: write_file + owner: engineering + risk_level: medium + side_effect: true + idempotency_required: false + timeout_seconds: 60 + allowed_agents: "boss,implement-agent,design-agent" + rollback: + strategy: delete_or_restore + script: .specify/scripts/bash/rollback-manager.sh + +level5_gate: + required_fields: + - id + - owner + - risk_level + - side_effect + - idempotency_required + - timeout_seconds + side_effecting_tools_require: + - idempotency_key + - audit_record + - rollback_strategy