gitea-admin/CASAN
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix T1+T4: wire rollback into pipeline; real Ollama telemetry in metrics

Browse Source 
T1 (H7): casan-step.mjs now calls rollback-manager.sh checkpoint before
overwriting plan.md at attempt-2, writes tx-id to plan.checkpoint.txid
sidecar, and executes rollback on REJECTED verdict. rollback-transactions.jsonl
records a real cp restore command. Adversarial test: checkpoint exists,
real cp command recorded, plan hash matches pre-overwrite content.

T4 (H6): casan-harness.sh exports CASAN_STEP_NAME=$ACTION_NAME before
agent-metrics.sh so nested model calls (model-call.py) and the provider-
cost-lookup.py query share the same step label. metrics.jsonl now writes
cost_source=provider_telemetry instead of word_count_estimate when a real
Ollama call is made within the same step. Adversarial test: verified with
CASAN_STEP_NAME=t4-telemetry-test end-to-end.

adversarial-harness-tests.sh: 40 → 44 PASS / 0 FAIL (+3 T1, +1 T4)
security-gate.sh: PASS=10 FAIL=0 SKIP=0 (verified, local ornith:9b)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
thanhnv
2026-07-01 11:25:50 +09:00
co-authored by Claude Sonnet 4.6
parent f74a5b6e42
commit a8edbea534
18 changed files with 284 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AINative_OKR_CASAN5/.specify/logs/cost/metrics.jsonl
+11 -10
View File
@@ -1,10 +1,11 @@
{"timestamp":"2026-07-01T02:01:42Z","trace_id":"7260140d-0ef6-4514-b03c-68de004f51f8","harness":"H6-agentops","agent":"demo.agent","step":"demo-step","status":"success","exit_code":0,"latency_ms":54,"retry_count":0,"input_tokens":6,"output_tokens":6,"total_tokens":12,"cost_estimate":0.00002400,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac","output_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac"}
{"timestamp":"2026-07-01T02:01:42Z","trace_id":"50a5bd13-1bc7-45f0-828d-42027d08c86e","harness":"H6-agentops","agent":"demo.agent","step":"step-1-srs","status":"success","exit_code":0,"latency_ms":212,"retry_count":0,"input_tokens":13,"output_tokens":13,"total_tokens":26,"cost_estimate":0.00005200,"cost_source":"word_count_estimate","hallucination_signals":4,"alerts":["hallucination-suspected"],"input_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57","output_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57"}
{"timestamp":"2026-07-01T02:01:43Z","trace_id":"3b4107b0-9e36-4ffc-9c0d-fbdf21d8199a","harness":"H6-agentops","agent":"demo.agent","step":"speckit.implement","status":"success","exit_code":0,"latency_ms":53,"retry_count":0,"input_tokens":6,"output_tokens":6,"total_tokens":2778,"cost_estimate":0.08334,"cost_source":"provider_telemetry","hallucination_signals":0,"alerts":[],"input_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac","output_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac"}
{"timestamp":"2026-07-01T02:01:43Z","trace_id":"2c0bc2f4-b444-45a9-b485-5c53093ea174","harness":"H6-agentops","agent":"demo.agent","step":"failing-step","status":"failed","exit_code":7,"latency_ms":192,"retry_count":0,"input_tokens":6,"output_tokens":0,"total_tokens":6,"cost_estimate":0.00001200,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":["execution-failed"],"input_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac","output_hash":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}
{"timestamp":"2026-07-01T02:01:44Z","trace_id":"ed183008-72fa-4427-92d2-a18c1741b9b1","harness":"H6-agentops","agent":"wrapper.demo","step":"wrapper-step","status":"success","exit_code":0,"latency_ms":51,"retry_count":0,"input_tokens":7,"output_tokens":7,"total_tokens":14,"cost_estimate":0.00002800,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116","output_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116"}
{"timestamp":"2026-07-01T02:01:46Z","trace_id":"c68fdf2f-d0aa-48df-be14-e2ed9748c6a5","harness":"H6-agentops","agent":"wrapper.demo","step":"wrapper-step","status":"success","exit_code":0,"latency_ms":185,"retry_count":0,"input_tokens":7,"output_tokens":7,"total_tokens":14,"cost_estimate":0.00002800,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116","output_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116"}
{"timestamp":"2026-07-01T02:01:55Z","trace_id":"50ce3010-3720-4ad5-9be9-40bc32ac177d","harness":"H6-agentops","agent":"unknown-agent","step":"unknown-step","status":"failed","exit_code":0,"latency_ms":208,"retry_count":0,"input_tokens":3,"output_tokens":0,"total_tokens":3,"cost_estimate":0.00000600,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":["execution-failed"],"input_hash":"cae0b3e41bdd7bf9fc5ab7f73d4422a65c3766f8097c90d952d07dd371605290","output_hash":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}
{"timestamp":"2026-07-01T02:01:56Z","trace_id":"7e0039e4-57db-4b3a-893f-653382dadbc5","harness":"H6-agentops","agent":"adv","step":"step-1-srs","status":"success","exit_code":0,"latency_ms":194,"retry_count":0,"input_tokens":13,"output_tokens":13,"total_tokens":26,"cost_estimate":0.00005200,"cost_source":"word_count_estimate","hallucination_signals":4,"alerts":["hallucination-suspected"],"input_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57","output_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57"}
{"timestamp":"2026-07-01T02:01:57Z","trace_id":"46dc7318-ab4b-4a11-960b-a0f9c883fc81","harness":"H6-agentops","agent":"adv","step":"step-1-srs","status":"success","exit_code":0,"latency_ms":178,"retry_count":0,"input_tokens":7,"output_tokens":7,"total_tokens":14,"cost_estimate":0.00002800,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"a97a0aba66ab15562e4d271fbcb2071c92d3662d99f2e2faac7f263ea35cfa0b","output_hash":"a97a0aba66ab15562e4d271fbcb2071c92d3662d99f2e2faac7f263ea35cfa0b"}
{"timestamp":"2026-07-01T02:02:02Z","trace_id":"5d94660f-b81c-463a-8495-2b1bacd18888","harness":"H6-agentops","agent":"unknown-agent","step":"unknown-step","status":"failed","exit_code":124,"latency_ms":2236,"retry_count":0,"input_tokens":1,"output_tokens":0,"total_tokens":1,"cost_estimate":0.00000200,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":["execution-failed"],"input_hash":"7d3f9b6284c6f36e77b425cac882e8fbbcc97a4727ec20790853076d0f463453","output_hash":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}
{"timestamp":"2026-07-01T02:23:34Z","trace_id":"faa37670-fa26-4b54-9ec4-013a66fb2c4a","harness":"H6-agentops","agent":"demo.agent","step":"demo-step","status":"success","exit_code":0,"latency_ms":53,"retry_count":0,"input_tokens":6,"output_tokens":6,"total_tokens":12,"cost_estimate":0.00002400,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac","output_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac"}
{"timestamp":"2026-07-01T02:23:35Z","trace_id":"01675f76-25f1-4f79-a55c-2571ecbc7314","harness":"H6-agentops","agent":"demo.agent","step":"step-1-srs","status":"success","exit_code":0,"latency_ms":193,"retry_count":0,"input_tokens":13,"output_tokens":13,"total_tokens":26,"cost_estimate":0.00005200,"cost_source":"word_count_estimate","hallucination_signals":4,"alerts":["hallucination-suspected"],"input_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57","output_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57"}
{"timestamp":"2026-07-01T02:23:35Z","trace_id":"a25e6809-60f7-4023-a059-f98dd12489fe","harness":"H6-agentops","agent":"demo.agent","step":"speckit.implement","status":"success","exit_code":0,"latency_ms":55,"retry_count":0,"input_tokens":6,"output_tokens":6,"total_tokens":2778,"cost_estimate":0.08334,"cost_source":"provider_telemetry","hallucination_signals":0,"alerts":[],"input_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac","output_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac"}
{"timestamp":"2026-07-01T02:23:35Z","trace_id":"47204bd3-200b-42d6-b724-9560d737ad6c","harness":"H6-agentops","agent":"demo.agent","step":"failing-step","status":"failed","exit_code":7,"latency_ms":175,"retry_count":0,"input_tokens":6,"output_tokens":0,"total_tokens":6,"cost_estimate":0.00001200,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":["execution-failed"],"input_hash":"2a5a257dc10475c9105ba981755390fb815f3102d3e6d4fc6f7fb161f7351bac","output_hash":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}
{"timestamp":"2026-07-01T02:23:38Z","trace_id":"a6dba904-47b9-4a62-bdd5-47c9f731fe65","harness":"H6-agentops","agent":"wrapper.demo","step":"wrapper-step","status":"success","exit_code":0,"latency_ms":50,"retry_count":0,"input_tokens":7,"output_tokens":7,"total_tokens":14,"cost_estimate":0.00002800,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116","output_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116"}
{"timestamp":"2026-07-01T02:23:40Z","trace_id":"46fc9bca-1d15-4982-b8f0-1316afd939ca","harness":"H6-agentops","agent":"wrapper.demo","step":"wrapper-step","status":"success","exit_code":0,"latency_ms":196,"retry_count":0,"input_tokens":7,"output_tokens":7,"total_tokens":14,"cost_estimate":0.00002800,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116","output_hash":"054cd5120725af9fdf9a8033f7b0f60d2d8ba3861926b00686d74700668e5116"}
{"timestamp":"2026-07-01T02:23:49Z","trace_id":"0a850cae-8333-436b-a056-a7845d23fc64","harness":"H6-agentops","agent":"unknown-agent","step":"write_code","status":"failed","exit_code":0,"latency_ms":195,"retry_count":0,"input_tokens":3,"output_tokens":0,"total_tokens":3,"cost_estimate":0.00000600,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":["execution-failed"],"input_hash":"cae0b3e41bdd7bf9fc5ab7f73d4422a65c3766f8097c90d952d07dd371605290","output_hash":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}
{"timestamp":"2026-07-01T02:23:50Z","trace_id":"c356f439-77bc-4815-b352-1e0e0187e150","harness":"H6-agentops","agent":"adv","step":"step-1-srs","status":"success","exit_code":0,"latency_ms":197,"retry_count":0,"input_tokens":13,"output_tokens":13,"total_tokens":26,"cost_estimate":0.00005200,"cost_source":"word_count_estimate","hallucination_signals":4,"alerts":["hallucination-suspected"],"input_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57","output_hash":"666dfe86cafeb8150bcc28d0b5cb2c43e1149dddaa126b5fbc8adee318f46d57"}
{"timestamp":"2026-07-01T02:23:51Z","trace_id":"bf633728-b454-4a4d-949f-b52eba6e0c99","harness":"H6-agentops","agent":"adv","step":"step-1-srs","status":"success","exit_code":0,"latency_ms":180,"retry_count":0,"input_tokens":7,"output_tokens":7,"total_tokens":14,"cost_estimate":0.00002800,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":[],"input_hash":"a97a0aba66ab15562e4d271fbcb2071c92d3662d99f2e2faac7f263ea35cfa0b","output_hash":"a97a0aba66ab15562e4d271fbcb2071c92d3662d99f2e2faac7f263ea35cfa0b"}
{"timestamp":"2026-07-01T02:23:56Z","trace_id":"bf9ae0e4-61cc-48e2-b6b6-a6add126c35c","harness":"H6-agentops","agent":"unknown-agent","step":"test_timeout","status":"failed","exit_code":124,"latency_ms":2206,"retry_count":0,"input_tokens":1,"output_tokens":0,"total_tokens":1,"cost_estimate":0.00000200,"cost_source":"word_count_estimate","hallucination_signals":0,"alerts":["execution-failed"],"input_hash":"7d3f9b6284c6f36e77b425cac882e8fbbcc97a4727ec20790853076d0f463453","output_hash":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}
{"timestamp":"2026-07-01T02:24:05Z","trace_id":"22168ecf-ed3a-47fe-b28b-262f589fd412","harness":"H6-agentops","agent":"unknown-agent","step":"t4-telemetry-test","status":"success","exit_code":0,"latency_ms":191,"retry_count":0,"input_tokens":2,"output_tokens":2,"total_tokens":210,"cost_estimate":0.0,"cost_source":"provider_telemetry","hallucination_signals":0,"alerts":[],"input_hash":"19f254d48ce072ae3832d0271898d40921875a1a870bf4389e67ef226ffc1cc6","output_hash":"19f254d48ce072ae3832d0271898d40921875a1a870bf4389e67ef226ffc1cc6"}