{"title":"Twenty synthetic agent task comparisons","date":"2026-10-01","status":"completed_exploratory_comparison","run_id":"7daf099b-99e2-4aa6-9c2d-e341e3c0007f","report_sha256":"3e5189d1efa624cd4384d47f30d1b8d8dce7fd1ba325e2081c3fd05eda3224fb","frozen_hashes":{"tasks":"3febf795723f61d1217104553cc73cdb2bcc6660896db9173048abd8713be9fb","oracles":"a8ba4516348b0ac1d7e35d9b623459ad337a635e310621e42749d9dd44ab2c9e","protocol":"8fe00c48a614d447f9294386ba26224ca3e81172a0a51e54a90ce46514da6ece","harness":"03cd60e49b58e6d8977151085fd48233cadcd42557258ef43e6329e3de903c80"},"requested_model":"gpt-6.1-sol","response_model":null,"requested_effort":"high","environment":{"node":"v22.18.0","libraries":{"csv-parse":"7.0.2","semver":"7.8.5"}},"task_count":20,"trial_count":40,"baseline_correct":19,"service_correct":20,"both_correct":19,"method":{"sample":"One exploratory pair per synthetic task; no within-task repetitions. Separate preliminary smoke pair retained outside these totals.","baseline":"Fresh local solver with Node, csv-parse and semver; no service call.","service":"Fresh solver with the same task plus a known public tool contract and a prepared parent-observed client. One fixed HTTP execution per service trial.","grading":"Same exact canonical JSON oracle check for both arms; incomplete, altered-packet, scope-flagged or unsuccessful-call trials fail. Differences are null unless both arms pass. The json-flatten-escapes prompt did not identify input.value explicitly. Its excluded local answer flattened the whole supplied object; this envelope-interpretation mismatch does not establish algorithmic failure or service correctness superiority. The frozen fixture and grade are retained; a future trial must clarify the wrapper.","accounting":"Native CLI completion usage includes full context and verification. Reasoning tokens are a subset of output, never added again. Prepared client setup is excluded.","interpretation":"Positive difference means the service used less; negative means more. Cache was observed, not controlled, so uncached-input differences do not establish service-caused or cash savings.","limits":"Not organic discovery, first installation, repeated-use economics, representative demand, or a general savings rate. Native stream did not report actual response model identity.","cash":"Actual cost and cash saved are unknown."},"metrics":{"input_tokens":{"eligible_pairs":19,"baseline_total":366478,"service_total":740393,"difference_total":-373915,"difference_median":-19654,"service_lower":0,"service_higher":19,"equal":0},"uncached_input_tokens":{"eligible_pairs":19,"baseline_total":221070,"service_total":248745,"difference_total":-27675,"difference_median":-443,"service_lower":2,"service_higher":17,"equal":0},"output_tokens":{"eligible_pairs":19,"baseline_total":1989,"service_total":1996,"difference_total":-7,"difference_median":-9,"service_lower":7,"service_higher":12,"equal":0},"completion_wall_ms":{"eligible_pairs":19,"baseline_total":187294,"service_total":333518,"difference_total":-146224,"difference_median":-7738,"service_lower":0,"service_higher":19,"equal":0}},"pairs":[{"task":"csv-quoted-rows","instruction":"Convert the CSV to JSON records with string values, including empty strings. Return the records array.","tool":"csv-to-json","order":["baseline","service"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19300,"cached_input_tokens":0,"uncached_input_tokens":19300,"output_tokens":98,"reasoning_output_tokens":28,"completion_wall_ms":9593,"process_wall_ms":10395,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"93d8ccb1d0811d938573f92acb57a30c5138ad71ec00bf8bc4b008591361101f","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38943,"cached_input_tokens":19200,"uncached_input_tokens":19743,"output_tokens":104,"reasoning_output_tokens":0,"completion_wall_ms":17331,"process_wall_ms":21353,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["4493d94c3cc091d280c6f6ca46dc715f1827b692a8457e3ce8342496c37bed05"],"answer_sha256":"93d8ccb1d0811d938573f92acb57a30c5138ad71ec00bf8bc4b008591361101f","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19643,"uncached_input_tokens":-443,"output_tokens":-6,"completion_wall_ms":-7738},"cash_saved":null},{"task":"csv-key-dedupe","instruction":"Deduplicate by id, preserving the first row for each id and original order. Return the output CSV string with LF and one final newline.","tool":"csv-dedupe","order":["service","baseline"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19302,"cached_input_tokens":0,"uncached_input_tokens":19302,"output_tokens":114,"reasoning_output_tokens":81,"completion_wall_ms":12898,"process_wall_ms":13599,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"ec9986eeb7c4c2e2afb0c8710b3c761babdf53ab909b667955372f380517eee6","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":39034,"cached_input_tokens":19328,"uncached_input_tokens":19706,"output_tokens":145,"reasoning_output_tokens":75,"completion_wall_ms":18663,"process_wall_ms":19428,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["cca78c3cd8df6754f12e5101df4bed7cb3af8330c486026bf5a9eeb19987b44d"],"answer_sha256":"ec9986eeb7c4c2e2afb0c8710b3c761babdf53ab909b667955372f380517eee6","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19732,"uncached_input_tokens":-404,"output_tokens":-31,"completion_wall_ms":-5765},"cash_saved":null},{"task":"csv-select-reorder","instruction":"Select note then code. Return RFC4180-quoted CSV with LF and one final newline.","tool":"csv-select-columns","order":["baseline","service"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19295,"cached_input_tokens":0,"uncached_input_tokens":19295,"output_tokens":252,"reasoning_output_tokens":202,"completion_wall_ms":15974,"process_wall_ms":16770,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"27a878647e70b8422b7fd59e7badc1682591dfeac31034004d2dd85dcbd2dda0","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38977,"cached_input_tokens":19328,"uncached_input_tokens":19649,"output_tokens":216,"reasoning_output_tokens":130,"completion_wall_ms":20130,"process_wall_ms":21626,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["b112c4fd8881d3942f7bce0761044b8e5ec70adb1f04144adc60ff8eed0a2546"],"answer_sha256":"27a878647e70b8422b7fd59e7badc1682591dfeac31034004d2dd85dcbd2dda0","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19682,"uncached_input_tokens":-354,"output_tokens":36,"completion_wall_ms":-4156},"cash_saved":null},{"task":"csv-group-count","instruction":"Count rows by the exact (region,kind) tuple in first-seen group order. Return the CSV with columns region,kind,count, LF and final newline.","tool":"csv-group-count","order":["service","baseline"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19305,"cached_input_tokens":18176,"uncached_input_tokens":1129,"output_tokens":180,"reasoning_output_tokens":137,"completion_wall_ms":12636,"process_wall_ms":13149,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"a7f9ff48586acaa00afc2e666948d880b0cdb4d0fe4be84c62fb76b5d3a95209","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":39014,"cached_input_tokens":19328,"uncached_input_tokens":19686,"output_tokens":172,"reasoning_output_tokens":93,"completion_wall_ms":18875,"process_wall_ms":20116,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["939dfc65bace7826dba11158f08b0d87ad7088864c99d4f2883ee2ba79a17f6f"],"answer_sha256":"a7f9ff48586acaa00afc2e666948d880b0cdb4d0fe4be84c62fb76b5d3a95209","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19709,"uncached_input_tokens":-18557,"output_tokens":8,"completion_wall_ms":-6239},"cash_saved":null},{"task":"json-pointer-escapes","instruction":"Evaluate this RFC6901 JSON Pointer and return the selected JSON value.","tool":"json-pointer-get","order":["baseline","service"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19292,"cached_input_tokens":0,"uncached_input_tokens":19292,"output_tokens":48,"reasoning_output_tokens":28,"completion_wall_ms":8095,"process_wall_ms":9109,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"01d875dbe512ada080a47883bc4e3596e983394a0bdaf2d03413a3339ce9ee86","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38916,"cached_input_tokens":19200,"uncached_input_tokens":19716,"output_tokens":73,"reasoning_output_tokens":0,"completion_wall_ms":15527,"process_wall_ms":16829,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["927b66f0335fd3d2abe16ecdbe149ec3cda64ca5cca9e864009c8bd0fb3b29ab"],"answer_sha256":"01d875dbe512ada080a47883bc4e3596e983394a0bdaf2d03413a3339ce9ee86","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19624,"uncached_input_tokens":-424,"output_tokens":-25,"completion_wall_ms":-7432},"cash_saved":null},{"task":"json-merge-patch","instruction":"Apply RFC7396 JSON Merge Patch and return the patched value.","tool":"json-merge-patch","order":["service","baseline"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19296,"cached_input_tokens":0,"uncached_input_tokens":19296,"output_tokens":77,"reasoning_output_tokens":34,"completion_wall_ms":9925,"process_wall_ms":10482,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"7ae9a90c1c01fca71f210f5266d91b154f24ffa5bf48c0b4326fd58daa3dffda","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38931,"cached_input_tokens":19200,"uncached_input_tokens":19731,"output_tokens":96,"reasoning_output_tokens":0,"completion_wall_ms":19159,"process_wall_ms":20345,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["a455e83415513835d68450fc82ea74039675411998633393885b440bad309338"],"answer_sha256":"7ae9a90c1c01fca71f210f5266d91b154f24ffa5bf48c0b4326fd58daa3dffda","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19635,"uncached_input_tokens":-435,"output_tokens":-19,"completion_wall_ms":-9234},"cash_saved":null},{"task":"json-flatten-escapes","instruction":"Flatten to an RFC6901 pointer map. Empty object is a leaf. Return that map.","tool":"json-flatten","order":["baseline","service"],"baseline":{"correct":false,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19281,"cached_input_tokens":0,"uncached_input_tokens":19281,"output_tokens":98,"reasoning_output_tokens":45,"completion_wall_ms":9085,"process_wall_ms":9705,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"5ad4f7d8b30aec794d82b1088305defd4937c74664c44586155af08324930ac0","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38929,"cached_input_tokens":37376,"uncached_input_tokens":1553,"output_tokens":104,"reasoning_output_tokens":0,"completion_wall_ms":16917,"process_wall_ms":17974,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["831f9d6de5bae8e54bfeada165f490f608e7c15e6d2cf52a1bd9314fa2038167"],"answer_sha256":"e2794b2173f03173e9f90c5dabdd96d73cc2b7448219d9138493a9b890ca1222","failure":null,"cash_cost":null},"both_correct":false,"difference_baseline_minus_service":{"input_tokens":null,"uncached_input_tokens":null,"output_tokens":null,"completion_wall_ms":null},"cash_saved":null},{"task":"json-stable-sort","instruction":"Stably sort ascending by team then name, comparing string values lexically. Return the items array.","tool":"json-array-sort-objects","order":["service","baseline"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19319,"cached_input_tokens":0,"uncached_input_tokens":19319,"output_tokens":99,"reasoning_output_tokens":29,"completion_wall_ms":9168,"process_wall_ms":10243,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"ded345d4c131100eb259607c4c5f0b36858606b6fc08bd7f56d5ab815193cdb0","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":39108,"cached_input_tokens":19328,"uncached_input_tokens":19780,"output_tokens":104,"reasoning_output_tokens":0,"completion_wall_ms":18544,"process_wall_ms":19759,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["514297d5036b5eb388835099ec884bd963d1648c1b4bf5be1c3541edeed5031b"],"answer_sha256":"ded345d4c131100eb259607c4c5f0b36858606b6fc08bd7f56d5ab815193cdb0","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19789,"uncached_input_tokens":-461,"output_tokens":-5,"completion_wall_ms":-9376},"cash_saved":null},{"task":"date-leap-crossing","instruction":"Add this number of calendar days in the proleptic Gregorian calendar. Return YYYY-MM-DD.","tool":"iso-date-add","order":["baseline","service"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19275,"cached_input_tokens":0,"uncached_input_tokens":19275,"output_tokens":92,"reasoning_output_tokens":67,"completion_wall_ms":8789,"process_wall_ms":9206,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"c5bd5a9c12de55bb7cc6e8d67c9ec15038f74ed1eb0710f175d0ff3d04a38a0b","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38927,"cached_input_tokens":37376,"uncached_input_tokens":1551,"output_tokens":53,"reasoning_output_tokens":0,"completion_wall_ms":13212,"process_wall_ms":14486,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["f2acc2b1e19a3cf0bce80e9795cdc013b600b6a44ddb52547dd86ef5b14c6c7e"],"answer_sha256":"c5bd5a9c12de55bb7cc6e8d67c9ec15038f74ed1eb0710f175d0ff3d04a38a0b","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19652,"uncached_input_tokens":17724,"output_tokens":39,"completion_wall_ms":-4423},"cash_saved":null},{"task":"date-month-clamp","instruction":"Add one calendar month, clamping to the final day of the target month. Return YYYY-MM-DD.","tool":"iso-date-add-months","order":["service","baseline"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19275,"cached_input_tokens":0,"uncached_input_tokens":19275,"output_tokens":64,"reasoning_output_tokens":39,"completion_wall_ms":7950,"process_wall_ms":8519,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"a5ba454cd9305d4de5f0c1c9040757e27881f8260741d465bfd1a519cbfc7fe9","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38956,"cached_input_tokens":37376,"uncached_input_tokens":1580,"output_tokens":59,"reasoning_output_tokens":0,"completion_wall_ms":17971,"process_wall_ms":19658,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["db07a2bc2a07d2e37cc85bda279208ece2b4124e0c18dcce8d70b1057b13e597"],"answer_sha256":"a5ba454cd9305d4de5f0c1c9040757e27881f8260741d465bfd1a519cbfc7fe9","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19681,"uncached_input_tokens":17695,"output_tokens":5,"completion_wall_ms":-10021},"cash_saved":null},{"task":"date-business-holiday","instruction":"Add five business days after the starting date; ISO weekdays6/7 and the listed holiday are not business days. Starting date is excluded. Return YYYY-MM-DD.","tool":"iso-date-business-day-add","order":["baseline","service"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19305,"cached_input_tokens":0,"uncached_input_tokens":19305,"output_tokens":81,"reasoning_output_tokens":56,"completion_wall_ms":8202,"process_wall_ms":8997,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"6bd8670b63f3fea402896eb088c5557d3e12f88bfcea0ed4f43f630fe76399eb","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":39156,"cached_input_tokens":19328,"uncached_input_tokens":19828,"output_tokens":59,"reasoning_output_tokens":0,"completion_wall_ms":16129,"process_wall_ms":17292,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["eeaa751971c333321afdb8db8f6078973977917530f142cca12be9f94f3a33a4"],"answer_sha256":"6bd8670b63f3fea402896eb088c5557d3e12f88bfcea0ed4f43f630fe76399eb","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19851,"uncached_input_tokens":-523,"output_tokens":22,"completion_wall_ms":-7927},"cash_saved":null},{"task":"exact-decimal-add","instruction":"Add the decimal strings exactly. Return the canonical decimal string, no unnecessary trailing zeroes.","tool":"decimal-add","order":["service","baseline"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19278,"cached_input_tokens":18176,"uncached_input_tokens":1102,"output_tokens":98,"reasoning_output_tokens":71,"completion_wall_ms":9259,"process_wall_ms":10308,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"347972feef85497c5ada9f075f341d38bc28223e05a974daafb1185cf6c3ce97","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38932,"cached_input_tokens":0,"uncached_input_tokens":38932,"output_tokens":78,"reasoning_output_tokens":21,"completion_wall_ms":16104,"process_wall_ms":17102,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["bb8f71bc54a9a1e8f0d8fb71954f4d11f5c8b6ff38fde2c5e9f8712c4db9fb60"],"answer_sha256":"347972feef85497c5ada9f075f341d38bc28223e05a974daafb1185cf6c3ce97","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19654,"uncached_input_tokens":-37830,"output_tokens":20,"completion_wall_ms":-6845},"cash_saved":null},{"task":"exact-decimal-subtract","instruction":"Subtract b from a exactly. Return the canonical decimal string.","tool":"decimal-subtract","order":["baseline","service"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19280,"cached_input_tokens":0,"uncached_input_tokens":19280,"output_tokens":67,"reasoning_output_tokens":45,"completion_wall_ms":9025,"process_wall_ms":9825,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"486881b68a9b84b17f8544257ac5fe7463ae68f082e8ba3b59f5f44e9480243f","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38907,"cached_input_tokens":19200,"uncached_input_tokens":19707,"output_tokens":76,"reasoning_output_tokens":18,"completion_wall_ms":16754,"process_wall_ms":18165,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["649368c61d7156394979587ae6450190f31b3de4860312fc290066529552227a"],"answer_sha256":"486881b68a9b84b17f8544257ac5fe7463ae68f082e8ba3b59f5f44e9480243f","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19627,"uncached_input_tokens":-427,"output_tokens":-9,"completion_wall_ms":-7729},"cash_saved":null},{"task":"text-line-endings","instruction":"Normalize CRLF and standalone CR line endings to LF. Return the text string.","tool":"text-line-endings-normalize","order":["service","baseline"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19273,"cached_input_tokens":0,"uncached_input_tokens":19273,"output_tokens":75,"reasoning_output_tokens":46,"completion_wall_ms":6493,"process_wall_ms":6872,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"7a684bf2c67c718bfc4d900a29e27e2e63a9dbfba541646b26fed6001cb8c9c7","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38898,"cached_input_tokens":19200,"uncached_input_tokens":19698,"output_tokens":100,"reasoning_output_tokens":35,"completion_wall_ms":17537,"process_wall_ms":18593,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["3dd72aa59c25b8eda60dd61640a7c2d9a379ad9188274580a361c479d70bfc5f"],"answer_sha256":"7a684bf2c67c718bfc4d900a29e27e2e63a9dbfba541646b26fed6001cb8c9c7","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19625,"uncached_input_tokens":-425,"output_tokens":-25,"completion_wall_ms":-11044},"cash_saved":null},{"task":"text-trailing-space","instruction":"Remove trailing spaces and tabs on each line while preserving leading spaces and original CRLF/LF endings. Return the text string.","tool":"text-trailing-whitespace-trim","order":["baseline","service"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19280,"cached_input_tokens":18176,"uncached_input_tokens":1104,"output_tokens":86,"reasoning_output_tokens":57,"completion_wall_ms":8532,"process_wall_ms":9084,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"cd5e6d9fbcad4166b343b617f2623aadf1e48cc67d70fba50369873461a3888d","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38934,"cached_input_tokens":37376,"uncached_input_tokens":1558,"output_tokens":111,"reasoning_output_tokens":31,"completion_wall_ms":18495,"process_wall_ms":19483,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["9cc50a7ba14d0f8d6c7ed6794dc3e3af52981e64e259fa7b413d4fd83ae37df0"],"answer_sha256":"cd5e6d9fbcad4166b343b617f2623aadf1e48cc67d70fba50369873461a3888d","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19654,"uncached_input_tokens":-454,"output_tokens":-25,"completion_wall_ms":-9963},"cash_saved":null},{"task":"text-leading-tabs","instruction":"Replace each leading tab with exactly three spaces. Keep internal tabs. Return the text string.","tool":"text-tabs-to-spaces","order":["service","baseline"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19279,"cached_input_tokens":18176,"uncached_input_tokens":1103,"output_tokens":57,"reasoning_output_tokens":29,"completion_wall_ms":7785,"process_wall_ms":8438,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"cb987563e93ac8ec39d7c313416a769526afb028f845335733cc5d327fd0845c","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38970,"cached_input_tokens":37376,"uncached_input_tokens":1594,"output_tokens":126,"reasoning_output_tokens":47,"completion_wall_ms":16875,"process_wall_ms":17903,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["c957abe761b381ff6df3f3439d0ba81c743d7877a0725967fea02474cf18cb00"],"answer_sha256":"cb987563e93ac8ec39d7c313416a769526afb028f845335733cc5d327fd0845c","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19691,"uncached_input_tokens":-491,"output_tokens":-69,"completion_wall_ms":-9090},"cash_saved":null},{"task":"semver-precedence","instruction":"Compare SemVer2.0 precedence. Return -1 if a<b,0 if equal,1 if a>b.","tool":"semver-compare","order":["baseline","service"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19287,"cached_input_tokens":18176,"uncached_input_tokens":1111,"output_tokens":49,"reasoning_output_tokens":31,"completion_wall_ms":8186,"process_wall_ms":8658,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"a65c5e19ff1f884bf9d18564fc7b6273809aa36ae3f8ac5c8b644c875a887452","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38941,"cached_input_tokens":37376,"uncached_input_tokens":1565,"output_tokens":73,"reasoning_output_tokens":0,"completion_wall_ms":18776,"process_wall_ms":19644,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["c7acaf31f473a465d72dcf188547080a17a14d8df8c2d89f4815edb2bc082216"],"answer_sha256":"a65c5e19ff1f884bf9d18564fc7b6273809aa36ae3f8ac5c8b644c875a887452","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19654,"uncached_input_tokens":-454,"output_tokens":-24,"completion_wall_ms":-10590},"cash_saved":null},{"task":"semver-caret-zero","instruction":"Check whether the stable version satisfies the npm caret range. Return a boolean.","tool":"semver-satisfies","order":["service","baseline"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19275,"cached_input_tokens":18176,"uncached_input_tokens":1099,"output_tokens":55,"reasoning_output_tokens":37,"completion_wall_ms":7407,"process_wall_ms":8486,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"1fe39d7c9bd6e0fcbb79038732f10071ed8588dbe59e2720fdca45116e58aa96","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38983,"cached_input_tokens":37376,"uncached_input_tokens":1607,"output_tokens":84,"reasoning_output_tokens":0,"completion_wall_ms":15103,"process_wall_ms":16484,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["336cd545fa0ae26f333a32307db2deb105a8d256b228735cb7ae38601fba9d45"],"answer_sha256":"1fe39d7c9bd6e0fcbb79038732f10071ed8588dbe59e2720fdca45116e58aa96","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19708,"uncached_input_tokens":-508,"output_tokens":-29,"completion_wall_ms":-7696},"cash_saved":null},{"task":"url-query-order","instruction":"Parse query pairs in order using application/x-www-form-urlencoded decoding. Return the array of objects with name and value.","tool":"url-query-parse","order":["baseline","service"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19295,"cached_input_tokens":18176,"uncached_input_tokens":1119,"output_tokens":86,"reasoning_output_tokens":28,"completion_wall_ms":8216,"process_wall_ms":8731,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"2293a6002833204fc43dd79c3d8dc5d36c79ca54a955c45a64b9f529dfbbb4c7","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38996,"cached_input_tokens":37376,"uncached_input_tokens":1620,"output_tokens":108,"reasoning_output_tokens":0,"completion_wall_ms":17734,"process_wall_ms":19205,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["fbc8ea6bbb1a16e9fc99c1d8f86971f2e8cff2e34570643c244d8e7adc1ece04"],"answer_sha256":"2293a6002833204fc43dd79c3d8dc5d36c79ca54a955c45a64b9f529dfbbb4c7","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19701,"uncached_input_tokens":-501,"output_tokens":-22,"completion_wall_ms":-9518},"cash_saved":null},{"task":"utf8-base64","instruction":"Encode the text as UTF-8 then standard padded Base64. Return the Base64 string.","tool":"utf8-base64","order":["service","baseline"],"baseline":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":19267,"cached_input_tokens":18176,"uncached_input_tokens":1091,"output_tokens":311,"reasoning_output_tokens":281,"completion_wall_ms":19161,"process_wall_ms":20003,"command_calls":0,"service_calls":0,"service_successes":0,"service_response_sha256":[],"answer_sha256":"015c7f2bdf26977e34876fcd718901e461817276cde034fc625613f81c090744","failure":null,"cash_cost":null},"service":{"correct":true,"requested_model":"gpt-6.1-sol","response_model":null,"input_tokens":38870,"cached_input_tokens":37376,"uncached_input_tokens":1494,"output_tokens":159,"reasoning_output_tokens":71,"completion_wall_ms":20599,"process_wall_ms":21439,"command_calls":1,"service_calls":1,"service_successes":1,"service_response_sha256":["0d7436483ec31bf71675a3f74c6a21675f89f2d725302c4b2aa06100a9fa4893"],"answer_sha256":"015c7f2bdf26977e34876fcd718901e461817276cde034fc625613f81c090744","failure":null,"cash_cost":null},"both_correct":true,"difference_baseline_minus_service":{"input_tokens":-19603,"uncached_input_tokens":-403,"output_tokens":152,"completion_wall_ms":-1438},"cash_saved":null}],"cash_saved":null}