thanhnvandClaude Sonnet 4.6
a8edbea534
T1 (H7): casan-step.mjs now calls rollback-manager.sh checkpoint before overwriting plan.md at attempt-2, writes tx-id to plan.checkpoint.txid sidecar, and executes rollback on REJECTED verdict. rollback-transactions.jsonl records a real cp restore command. Adversarial test: checkpoint exists, real cp command recorded, plan hash matches pre-overwrite content. T4 (H6): casan-harness.sh exports CASAN_STEP_NAME=$ACTION_NAME before agent-metrics.sh so nested model calls (model-call.py) and the provider- cost-lookup.py query share the same step label. metrics.jsonl now writes cost_source=provider_telemetry instead of word_count_estimate when a real Ollama call is made within the same step. Adversarial test: verified with CASAN_STEP_NAME=t4-telemetry-test end-to-end. adversarial-harness-tests.sh: 40 → 44 PASS / 0 FAIL (+3 T1, +1 T4) security-gate.sh: PASS=10 FAIL=0 SKIP=0 (verified, local ornith:9b) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
12 lines
5.8 KiB
JSON
12 lines
5.8 KiB
JSON
{"timestamp":"2026-07-01T02:23:34Z","trace_id":"faa37670-fa26-4b54-9ec4-013a66fb2c4a","harness":"H6-agentops","agent":"demo.agent","step":"demo-step","status":"success","exit_code":0,"latency_ms":53,"retry_count":0,"input_tokens":6,"output_tokens":6,"total_tokens":12,"cost_estimate":0.00002400,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac","output_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac"}
|
|
{"timestamp":"2026-07-01T02:23:35Z","trace_id":"01675f76-25f1-4f79-a55c-2571ecbc7314","harness":"H6-agentops","agent":"demo.agent","step":"step-1-srs","status":"success","exit_code":0,"latency_ms":193,"retry_count":0,"input_tokens":13,"output_tokens":13,"total_tokens":26,"cost_estimate":0.00005200,"cost_source":"word_count_estimate","hallucination_signals":4,"alerts":["hallucination-suspected"],"input_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57","output_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57"}
|
|
{"timestamp":"2026-07-01T02:23:35Z","trace_id":"a25e6809-60f7-4023-a059-f98dd12489fe","harness":"H6-agentops","agent":"demo.agent","step":"speckit.implement","status":"success","exit_code":0,"latency_ms":55,"retry_count":0,"input_tokens":6,"output_tokens":6,"total_tokens":2778,"cost_estimate":0.08334,"cost_source":"provider_telemetry","hallucination_signals":0,"alerts":[],"input_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac","output_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac"}
|
|
{"timestamp":"2026-07-01T02:23:35Z","trace_id":"47204bd3-200b-42d6-b724-9560d737ad6c","harness":"H6-agentops","agent":"demo.agent","step":"failing-step","status":"failed","exit_code":7,"latency_ms":175,"retry_count":0,"input_tokens":6,"output_tokens":0,"total_tokens":6,"cost_estimate":0.00001200,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":["execution-failed"],"input_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac","output_hash":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}
|
|
{"timestamp":"2026-07-01T02:23:38Z","trace_id":"a6dba904-47b9-4a62-bdd5-47c9f731fe65","harness":"H6-agentops","agent":"wrapper.demo","step":"wrapper-step","status":"success","exit_code":0,"latency_ms":50,"retry_count":0,"input_tokens":7,"output_tokens":7,"total_tokens":14,"cost_estimate":0.00002800,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116","output_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116"}
|
|
{"timestamp":"2026-07-01T02:23:40Z","trace_id":"46fc9bca-1d15-4982-b8f0-1316afd939ca","harness":"H6-agentops","agent":"wrapper.demo","step":"wrapper-step","status":"success","exit_code":0,"latency_ms":196,"retry_count":0,"input_tokens":7,"output_tokens":7,"total_tokens":14,"cost_estimate":0.00002800,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116","output_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116"}
|
|
{"timestamp":"2026-07-01T02:23:49Z","trace_id":"0a850cae-8333-436b-a056-a7845d23fc64","harness":"H6-agentops","agent":"unknown-agent","step":"write_code","status":"failed","exit_code":0,"latency_ms":195,"retry_count":0,"input_tokens":3,"output_tokens":0,"total_tokens":3,"cost_estimate":0.00000600,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":["execution-failed"],"input_hash":"cae0b3e41bdd7bf9fc5ab7f73d4422a65c3766f8097c90d952d07dd371605290","output_hash":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}
|
|
{"timestamp":"2026-07-01T02:23:50Z","trace_id":"c356f439-77bc-4815-b352-1e0e0187e150","harness":"H6-agentops","agent":"adv","step":"step-1-srs","status":"success","exit_code":0,"latency_ms":197,"retry_count":0,"input_tokens":13,"output_tokens":13,"total_tokens":26,"cost_estimate":0.00005200,"cost_source":"word_count_estimate","hallucination_signals":4,"alerts":["hallucination-suspected"],"input_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57","output_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57"}
|
|
{"timestamp":"2026-07-01T02:23:51Z","trace_id":"bf633728-b454-4a4d-949f-b52eba6e0c99","harness":"H6-agentops","agent":"adv","step":"step-1-srs","status":"success","exit_code":0,"latency_ms":180,"retry_count":0,"input_tokens":7,"output_tokens":7,"total_tokens":14,"cost_estimate":0.00002800,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"a97a0aba66ab15562e4d271fbcb2071c92d3662d99f2e2faac7f263ea35cfa0b","output_hash":"a97a0aba66ab15562e4d271fbcb2071c92d3662d99f2e2faac7f263ea35cfa0b"}
|
|
{"timestamp":"2026-07-01T02:23:56Z","trace_id":"bf9ae0e4-61cc-48e2-b6b6-a6add126c35c","harness":"H6-agentops","agent":"unknown-agent","step":"test_timeout","status":"failed","exit_code":124,"latency_ms":2206,"retry_count":0,"input_tokens":1,"output_tokens":0,"total_tokens":1,"cost_estimate":0.00000200,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":["execution-failed"],"input_hash":"7d3f9b6284c6f36e77b425cac882e8fbbcc97a4727ec20790853076d0f463453","output_hash":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}
|
|
{"timestamp":"2026-07-01T02:24:05Z","trace_id":"22168ecf-ed3a-47fe-b28b-262f589fd412","harness":"H6-agentops","agent":"unknown-agent","step":"t4-telemetry-test","status":"success","exit_code":0,"latency_ms":191,"retry_count":0,"input_tokens":2,"output_tokens":2,"total_tokens":210,"cost_estimate":0.0,"cost_source":"provider_telemetry","hallucination_signals":0,"alerts":[],"input_hash":"19f254d48ce072ae3832d0271898d40921875a1a870bf4389e67ef226ffc1cc6","output_hash":"19f254d48ce072ae3832d0271898d40921875a1a870bf4389e67ef226ffc1cc6"}
|