update first - 84
This commit is contained in:
thanhnv
@@ -0,0 +1,4 @@
|
||||
{"timestamp":"2026-06-29T17:01:08Z","trace_id":"98e40b76-6db7-4357-83c4-b879308cd645","severity":"WARN","resource":{"service.name":"demo.agent","service.version":"1.0.0"},"body":{"message":"Alert triggered: hallucination-suspected","alert.type":"hallucination-suspected","step.name":"step-1-srs"},"attributes":{"latency_ms":220,"status":"success"}}
|
||||
{"timestamp":"2026-06-29T17:01:09Z","trace_id":"45331383-aaed-47b7-bfa1-58eba82adcd3","severity":"WARN","resource":{"service.name":"demo.agent","service.version":"1.0.0"},"body":{"message":"Alert triggered: execution-failed","alert.type":"execution-failed","step.name":"failing-step"},"attributes":{"latency_ms":198,"status":"failed"}}
|
||||
{"timestamp":"2026-06-29T17:01:22Z","trace_id":"997d7a8d-7d82-4b85-acd5-7b95ceaa0aaf","severity":"WARN","resource":{"service.name":"unknown-agent","service.version":"1.0.0"},"body":{"message":"Alert triggered: execution-failed","alert.type":"execution-failed","step.name":"unknown-step"},"attributes":{"latency_ms":210,"status":"failed"}}
|
||||
{"timestamp":"2026-06-29T17:01:23Z","trace_id":"d5857d18-83ac-4dbf-8d47-aebb4549255d","severity":"WARN","resource":{"service.name":"adv","service.version":"1.0.0"},"body":{"message":"Alert triggered: hallucination-suspected","alert.type":"hallucination-suspected","step.name":"step-1-srs"},"attributes":{"latency_ms":205,"status":"success"}}
|
||||
@@ -0,0 +1,86 @@
|
||||
version: 1.0
|
||||
description: AgentOps Alert Rules
|
||||
|
||||
alerts:
|
||||
|
||||
- id: ALERT-001
|
||||
name: High Latency
|
||||
condition: latency_ms > 5000
|
||||
severity: medium
|
||||
action:
|
||||
- log
|
||||
- notify
|
||||
|
||||
- id: ALERT-002
|
||||
name: High Retry
|
||||
condition: retry_count > 2
|
||||
severity: medium
|
||||
action:
|
||||
- log
|
||||
- notify
|
||||
|
||||
- id: ALERT-003
|
||||
name: Failure Detected
|
||||
condition: status == "failed"
|
||||
severity: high
|
||||
action:
|
||||
- log
|
||||
- notify
|
||||
- escalate
|
||||
|
||||
- id: ALERT-004
|
||||
name: High Cost Spike
|
||||
condition: cost_estimate > 0.05
|
||||
severity: medium
|
||||
action:
|
||||
- log
|
||||
|
||||
- id: ALERT-005
|
||||
name: Hallucination Risk
|
||||
condition: output contains ("maybe" or "might be incorrect")
|
||||
severity: low
|
||||
action:
|
||||
- log
|
||||
- flag
|
||||
|
||||
- id: ALERT-006
|
||||
name: Token Overuse
|
||||
condition: total_tokens > 5000
|
||||
severity: medium
|
||||
action:
|
||||
- log
|
||||
- notify
|
||||
|
||||
notification:
|
||||
|
||||
channels:
|
||||
|
||||
- type: log
|
||||
path: ".specify/agentops/alerts.log"
|
||||
|
||||
- type: console
|
||||
enabled: true
|
||||
|
||||
# future integration
|
||||
# - type: slack
|
||||
# webhook: "https://hooks.slack.com/..."
|
||||
|
||||
escalation:
|
||||
|
||||
enabled: true
|
||||
|
||||
rules:
|
||||
- severity: high
|
||||
action: immediate_alert
|
||||
|
||||
- severity: medium
|
||||
action: batched_alert
|
||||
|
||||
- severity: low
|
||||
action: log_only
|
||||
|
||||
level4_gate:
|
||||
alert_log_must_exist: true
|
||||
failed_step_must_emit_alert: true
|
||||
retry_threshold_must_emit_alert: true
|
||||
cost_spike_must_emit_alert: true
|
||||
@@ -0,0 +1,76 @@
|
||||
version: 1.0
|
||||
description: CASAN H6 per-step hallucination signal tracking
|
||||
|
||||
tracking:
|
||||
enabled: true
|
||||
method: cross_reference_check
|
||||
log_path: .specify/logs/cost/metrics.jsonl
|
||||
field_name: hallucination_signals
|
||||
|
||||
steps_with_tracking:
|
||||
- step: step-1-srs
|
||||
reference_docs:
|
||||
- docs/input/okr-requirement.md
|
||||
signals:
|
||||
- ungrounded_feature_claim
|
||||
- invented_requirement
|
||||
- undefined_acronym
|
||||
keywords_indicating_hallucination:
|
||||
- "I assume"
|
||||
- "typically"
|
||||
- "usually includes"
|
||||
- "standard practice suggests"
|
||||
|
||||
- step: step-3-spec
|
||||
reference_docs:
|
||||
- docs/output/ipa-docs/srs/
|
||||
- docs/output/ipa-docs/bd/
|
||||
signals:
|
||||
- invented_api_endpoint
|
||||
- schema_mismatch
|
||||
- undefined_term
|
||||
- ungrounded_business_rule
|
||||
keywords_indicating_hallucination:
|
||||
- "assumed from context"
|
||||
- "common pattern would be"
|
||||
- "likely the intent is"
|
||||
|
||||
- step: step-6-plan
|
||||
reference_docs:
|
||||
- docs/output/specs/
|
||||
signals:
|
||||
- invented_dependency
|
||||
- wrong_module_assignment
|
||||
- unspecified_library
|
||||
keywords_indicating_hallucination:
|
||||
- "might need"
|
||||
- "could potentially use"
|
||||
- "I believe"
|
||||
|
||||
- step: step-10-implement
|
||||
reference_docs:
|
||||
- docs/output/specs/
|
||||
signals:
|
||||
- undocumented_endpoint
|
||||
- wrong_field_type
|
||||
- missing_required_field
|
||||
- hardcoded_value_not_in_spec
|
||||
keywords_indicating_hallucination:
|
||||
- "// TODO: confirm"
|
||||
- "// assumed"
|
||||
- "// placeholder"
|
||||
|
||||
- step: step-11-review-code
|
||||
reference_docs:
|
||||
- docs/output/specs/
|
||||
- backend/
|
||||
- frontend/
|
||||
signals:
|
||||
- false_security_clearance
|
||||
- invented_test_result
|
||||
- unverified_compliance_claim
|
||||
|
||||
thresholds:
|
||||
warn: 3
|
||||
block: 7
|
||||
note: "warn=log warning; block=force retry of step"
|
||||
@@ -0,0 +1,52 @@
|
||||
version: 1.0
|
||||
description: CASAN H6 AgentOps metrics and scoring configuration
|
||||
|
||||
collection:
|
||||
enabled: true
|
||||
trace_format: json
|
||||
metrics_format: jsonl
|
||||
trace_dir: ".specify/logs/trace"
|
||||
metrics_log: ".specify/logs/cost/metrics.jsonl"
|
||||
alert_log: ".specify/agentops/alerts.log"
|
||||
|
||||
dimensions:
|
||||
- trace_id
|
||||
- timestamp
|
||||
- agent
|
||||
- step
|
||||
- status
|
||||
- exit_code
|
||||
- latency_ms
|
||||
- retry_count
|
||||
- input_tokens
|
||||
- output_tokens
|
||||
- total_tokens
|
||||
- cost_estimate
|
||||
- alerts
|
||||
- input_hash
|
||||
- output_hash
|
||||
|
||||
cost_model:
|
||||
default_cost_per_1k_tokens_usd: 0.002
|
||||
override_env: CASAN_COST_PER_1K
|
||||
|
||||
thresholds:
|
||||
latency_alert_ms: 5000
|
||||
retry_alert_threshold: 2
|
||||
token_alert_threshold: 5000
|
||||
failure_alert: true
|
||||
|
||||
level4_gate:
|
||||
required: true
|
||||
minimum_evidence:
|
||||
traces_per_pipeline_step: 1
|
||||
metrics_log_present: true
|
||||
alert_rules_present: true
|
||||
failure_status_recorded: true
|
||||
|
||||
level5_readiness:
|
||||
required_additions:
|
||||
- drift_detection_against_golden_runs
|
||||
- model_fallback_routing
|
||||
- live_dashboard
|
||||
- business_outcome_metrics
|
||||
@@ -0,0 +1,36 @@
|
||||
version: 1.0
|
||||
description: CASAN H2 Tool rate limits — prevent runaway agent loops
|
||||
|
||||
limits:
|
||||
- tool: deploy
|
||||
max_per_pipeline: 3
|
||||
window: pipeline_run
|
||||
note: "Max 3 deploy attempts per pipeline run before escalation"
|
||||
|
||||
- tool: migration
|
||||
max_per_pipeline: 5
|
||||
window: pipeline_run
|
||||
note: "Max 5 migration runs — includes retries"
|
||||
|
||||
- tool: db_write
|
||||
max_per_pipeline: 10
|
||||
window: pipeline_run
|
||||
|
||||
- tool: write_code
|
||||
max_per_pipeline: 50
|
||||
window: pipeline_run
|
||||
note: "50 write_code calls covers BE+FE implementation + up to 5 retry cycles"
|
||||
|
||||
- tool: external_api
|
||||
max_per_hour: 100
|
||||
window: rolling_60min
|
||||
|
||||
- tool: agent_step
|
||||
max_per_pipeline: 200
|
||||
window: pipeline_run
|
||||
note: "Covers all 13 steps × max retries"
|
||||
|
||||
enforcement:
|
||||
on_limit_exceeded: deny
|
||||
log_denials: true
|
||||
log_path: .specify/logs/audit/rate-limit-denials.jsonl
|
||||
@@ -0,0 +1,62 @@
|
||||
version: 1.0
|
||||
description: Agent Tracking & Tracing Configuration
|
||||
|
||||
tracking:
|
||||
|
||||
enabled: true
|
||||
|
||||
trace:
|
||||
|
||||
fields:
|
||||
- trace_id
|
||||
- timestamp
|
||||
- user_input
|
||||
- processed_input
|
||||
- output
|
||||
- latency_ms
|
||||
- tokens_input
|
||||
- tokens_output
|
||||
- total_tokens
|
||||
- cost_estimate
|
||||
- retry_count
|
||||
- status
|
||||
- error
|
||||
|
||||
correlation:
|
||||
|
||||
enable: true
|
||||
key: trace_id
|
||||
|
||||
stages:
|
||||
|
||||
- name: input_received
|
||||
log: true
|
||||
|
||||
- name: security_checked
|
||||
log: true
|
||||
|
||||
- name: governance_checked
|
||||
log: true
|
||||
|
||||
- name: llm_called
|
||||
log: true
|
||||
|
||||
- name: output_filtered
|
||||
log: true
|
||||
|
||||
storage:
|
||||
|
||||
type: file
|
||||
path: ".specify/logs/trace/"
|
||||
|
||||
format: json
|
||||
|
||||
retention:
|
||||
|
||||
days: 30
|
||||
|
||||
logging:
|
||||
|
||||
level: info
|
||||
include_input: true
|
||||
include_output: true
|
||||
Reference in New Issue
Block a user