- dirname: 2025-02-25-20-23-07--gemini-pro
  test_cases: 225
  model: Gemini 2.0 Pro exp-02-05
  edit_format: whole
  commit_hash: 2fccd47
  pass_rate_1: 20.4
  pass_rate_2: 35.6
  pass_num_1: 46
  pass_num_2: 80
  percent_cases_well_formed: 100.0
  error_outputs: 430
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 13
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 5
  total_tests: 225
  command: aider --model gemini/gemini-2.0-pro-exp-02-05
  date: 2025-02-25
  versions: 0.75.2.dev
  seconds_per_case: 34.8
  total_cost: 0.0000

- dirname: 2024-12-21-18-41-18--polyglot-gpt-4o-mini
  test_cases: 225
  model: gpt-4o-mini-2024-07-18
  edit_format: whole
  commit_hash: a755079-dirty
  pass_rate_1: 0.9
  pass_rate_2: 3.6
  pass_num_1: 2
  pass_num_2: 8
  percent_cases_well_formed: 100.0
  error_outputs: 0
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 36
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 3
  total_tests: 225
  command: aider --model gpt-4o-mini-2024-07-18
  date: 2024-12-21
  versions: 0.69.2.dev
  seconds_per_case: 17.3
  total_cost: 0.3236

- dirname: 2025-01-17-19-44-33--sonnet-baseline-jan-17
  test_cases: 225
  model: claude-3-5-sonnet-20241022
  edit_format: diff
  commit_hash: 6451d59
  pass_rate_1: 22.2
  pass_rate_2: 51.6
  pass_num_1: 50
  pass_num_2: 116
  percent_cases_well_formed: 99.6
  error_outputs: 2
  num_malformed_responses: 1
  num_with_malformed_responses: 1
  user_asks: 11
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 8
  total_tests: 225
  command: aider --model claude-3-5-sonnet-20241022
  date: 2025-01-17
  versions: 0.71.2.dev
  seconds_per_case: 21.4
  total_cost: 14.4063
  
- dirname: 2024-12-30-20-57-12--gpt-4o-2024-11-20-ex-as-sys
  test_cases: 225
  model: gpt-4o-2024-11-20
  edit_format: diff
  commit_hash: 09ee197-dirty
  pass_rate_1: 4.9
  pass_rate_2: 18.2
  pass_num_1: 11
  pass_num_2: 41
  percent_cases_well_formed: 95.1
  error_outputs: 12
  num_malformed_responses: 12
  num_with_malformed_responses: 11
  user_asks: 53
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 12
  total_tests: 225
  command: aider --model gpt-4o-2024-11-20
  date: 2024-12-30
  versions: 0.70.1.dev
  seconds_per_case: 12.1
  total_cost: 6.7351
  
- dirname: 2024-12-30-20-44-54--gpt4o-ex-as-sys-clean-prompt
  test_cases: 225
  model: gpt-4o-2024-08-06
  edit_format: diff
  commit_hash: 09ee197-dirty
  pass_rate_1: 4.9
  pass_rate_2: 23.1
  pass_num_1: 11
  pass_num_2: 52
  percent_cases_well_formed: 94.2
  error_outputs: 21
  num_malformed_responses: 21
  num_with_malformed_responses: 13
  user_asks: 65
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 3
  total_tests: 225
  command: aider --model gpt-4o-2024-08-06
  date: 2024-12-30
  versions: 0.70.1.dev
  seconds_per_case: 16.0
  total_cost: 7.0286
  
- dirname: 2024-12-21-19-23-03--polyglot-o1-hard-diff
  test_cases: 224
  model: o1-2024-12-17 (high)
  edit_format: diff
  commit_hash: a755079-dirty
  pass_rate_1: 23.7
  pass_rate_2: 61.7
  pass_num_1: 53
  pass_num_2: 139
  percent_cases_well_formed: 91.5
  error_outputs: 25
  num_malformed_responses: 24
  num_with_malformed_responses: 19
  user_asks: 16
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 2
  total_tests: 225
  command: aider --model openrouter/openai/o1
  date: 2024-12-21
  versions: 0.69.2.dev
  seconds_per_case: 133.2
  total_cost: 186.4958

- dirname: 2024-12-21-20-56-21--polyglot-deepseek-diff
  test_cases: 225
  model: DeepSeek Chat V2.5
  edit_format: diff
  commit_hash: a755079-dirty
  pass_rate_1: 5.3
  pass_rate_2: 17.8
  pass_num_1: 12
  pass_num_2: 40
  percent_cases_well_formed: 92.9
  error_outputs: 42
  num_malformed_responses: 37
  num_with_malformed_responses: 16
  user_asks: 23
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 5
  test_timeouts: 5
  total_tests: 225
  command: aider --model deepseek/deepseek-chat
  date: 2024-12-21
  versions: 0.69.2.dev
  seconds_per_case: 184.0
  total_cost: 0.5101

- dirname: 2024-12-21-21-46-27--polyglot-haiku-diff
  test_cases: 225
  model: claude-3-5-haiku-20241022
  edit_format: diff
  commit_hash: a755079-dirty
  pass_rate_1: 7.1
  pass_rate_2: 28.0
  pass_num_1: 16
  pass_num_2: 63
  percent_cases_well_formed: 91.1
  error_outputs: 31
  num_malformed_responses: 30
  num_with_malformed_responses: 20
  user_asks: 13
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 9
  total_tests: 225
  command: aider --model claude-3-5-haiku-20241022
  date: 2024-12-21
  versions: 0.69.2.dev
  seconds_per_case: 31.8
  total_cost: 6.0583

- dirname: 2024-12-22-13-22-32--polyglot-qwen-diff
  test_cases: 225
  model: Qwen2.5-Coder-32B-Instruct
  edit_format: diff
  commit_hash: 6d7e8be-dirty
  pass_rate_1: 4.4
  pass_rate_2: 8.0
  pass_num_1: 10
  pass_num_2: 18
  percent_cases_well_formed: 71.6
  error_outputs: 158
  num_malformed_responses: 148
  num_with_malformed_responses: 64
  user_asks: 132
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 2
  total_tests: 225
  command: "aider --model openai/Qwen/Qwen2.5-Coder-32B-Instruct # via hyperbolic"
  date: 2024-12-22
  versions: 0.69.2.dev
  seconds_per_case: 84.4
  total_cost: 0.0000

- dirname: 2024-12-22-21-26-35--polyglot-o1mini-whole
  test_cases: 225
  model: o1-mini-2024-09-12
  edit_format: whole
  commit_hash: 37df899
  pass_rate_1: 5.8
  pass_rate_2: 32.9
  pass_num_1: 13
  pass_num_2: 74
  percent_cases_well_formed: 96.9
  error_outputs: 8
  num_malformed_responses: 8
  num_with_malformed_responses: 7
  user_asks: 27
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 3
  total_tests: 225
  command: aider --model o1-mini
  date: 2024-12-22
  versions: 0.69.2.dev
  seconds_per_case: 34.7
  total_cost: 18.5770

- dirname: 2024-12-22-18-43-25--gemini-exp-1206-polyglot-whole-2
  test_cases: 225
  model: gemini-exp-1206
  edit_format: whole
  commit_hash: b1bc2f8
  pass_rate_1: 19.6
  pass_rate_2: 38.2
  pass_num_1: 44
  pass_num_2: 86
  percent_cases_well_formed: 98.2
  error_outputs: 8
  num_malformed_responses: 8
  num_with_malformed_responses: 4
  user_asks: 32
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 9
  total_tests: 225
  command: aider --model gemini/gemini-exp-1206
  date: 2024-12-22
  versions: 0.69.2.dev
  seconds_per_case: 45.5
  total_cost: 0.0000
  
- dirname: 2024-12-22-20-08-13--gemini-2.0-flash-exp-polyglot-whole
  test_cases: 225
  model: gemini-2.0-flash-exp
  edit_format: whole
  commit_hash: b1bc2f8
  pass_rate_1: 11.6
  pass_rate_2: 22.2
  pass_num_1: 26
  pass_num_2: 50
  percent_cases_well_formed: 100.0
  error_outputs: 1
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 9
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 8
  total_tests: 225
  command: aider --model gemini/gemini-2.0-flash-exp
  date: 2024-12-22
  versions: 0.69.2.dev
  seconds_per_case: 12.2
  total_cost: 0.0000
  
- dirname: 2024-12-23-01-11-56--yi-test
  test_cases: 225
  model: yi-lightning
  edit_format: whole
  commit_hash: 2b1625e
  pass_rate_1: 5.8
  pass_rate_2: 12.9
  pass_num_1: 13
  pass_num_2: 29
  percent_cases_well_formed: 92.9
  error_outputs: 87
  num_malformed_responses: 72
  num_with_malformed_responses: 16
  user_asks: 107
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 6
  total_tests: 225
  command: aider --model openai/yi-lightning
  date: 2024-12-23
  versions: 0.69.2.dev
  seconds_per_case: 146.7
  total_cost: 0.0000

- dirname: 2024-12-25-13-31-51--deepseekv3preview-diff2
  test_cases: 225
  model: DeepSeek Chat V3 (prev)
  edit_format: diff
  commit_hash: 0a23c4a-dirty
  pass_rate_1: 22.7
  pass_rate_2: 48.4
  pass_num_1: 51
  pass_num_2: 109
  percent_cases_well_formed: 98.7
  error_outputs: 7
  num_malformed_responses: 7
  num_with_malformed_responses: 3
  user_asks: 19
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 8
  total_tests: 225
  command: aider --model deepseek/deepseek-chat
  date: 2024-12-25
  versions: 0.69.2.dev
  seconds_per_case: 34.8
  total_cost: 0.3369

- dirname: 2024-12-26-00-55-20--Qwen2.5-Coder-32B-Instruct
  test_cases: 225
  model: Qwen2.5-Coder-32B-Instruct
  edit_format: whole
  commit_hash: b51768b0
  pass_rate_1: 4.9
  pass_rate_2: 16.4
  pass_num_1: 11
  pass_num_2: 37
  percent_cases_well_formed: 99.6
  error_outputs: 1
  num_malformed_responses: 1
  num_with_malformed_responses: 1
  user_asks: 33
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 6
  total_tests: 225
  command: aider --model openai/Qwen2.5-Coder-32B-Instruct
  date: 2024-12-26
  versions: 0.69.2.dev
  seconds_per_case: 42.0
  total_cost: 0.0000

- dirname: 2025-01-13-18-17-25--codestral-whole2
  test_cases: 225
  model: Codestral 25.01
  edit_format: whole
  commit_hash: 0cba898-dirty
  pass_rate_1: 4.0
  pass_rate_2: 11.1
  pass_num_1: 9
  pass_num_2: 25
  percent_cases_well_formed: 100.0
  error_outputs: 0
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 47
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 4
  total_tests: 225
  command: aider --model mistral/codestral-latest
  date: 2025-01-13
  versions: 0.71.2.dev
  seconds_per_case: 9.3
  total_cost: 1.9834

- dirname: 2025-01-20-19-11-38--ds-turns-upd-cur-msgs-fix-with-summarizer
  test_cases: 225
  model: DeepSeek R1
  edit_format: diff
  commit_hash: 5650697-dirty
  pass_rate_1: 26.7
  pass_rate_2: 56.9
  pass_num_1: 60
  pass_num_2: 128
  percent_cases_well_formed: 96.9
  error_outputs: 8
  num_malformed_responses: 7
  num_with_malformed_responses: 7
  user_asks: 15
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 5
  total_tests: 225
  command: aider --model deepseek/deepseek-reasoner
  date: 2025-01-20
  versions: 0.71.2.dev
  seconds_per_case: 113.7
  total_cost: 5.4193

- dirname: 2025-01-23-19-14-48--r1-architect-sonnet
  test_cases: 225
  model: DeepSeek R1 + claude-3-5-sonnet-20241022
  edit_format: architect
  commit_hash: 05a77c7
  editor_model: claude-3-5-sonnet-20241022
  editor_edit_format: editor-diff
  pass_rate_1: 27.1
  pass_rate_2: 64.0
  pass_num_1: 61
  pass_num_2: 144
  percent_cases_well_formed: 100.0
  error_outputs: 2
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 392
  lazy_comments: 6
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 5
  total_tests: 225
  command: aider --architect --model r1 --editor-model sonnet
  date: 2025-01-23
  versions: 0.72.3.dev
  seconds_per_case: 251.6
  total_cost: 13.2933

- dirname: 2025-01-28-16-00-03--qwen-max-2025-01-25-polyglot-diff
  test_cases: 225
  model: qwen-max-2025-01-25
  edit_format: diff
  commit_hash: ae7d459
  pass_rate_1: 9.3
  pass_rate_2: 21.8
  pass_num_1: 21
  pass_num_2: 49
  percent_cases_well_formed: 90.2
  error_outputs: 46
  num_malformed_responses: 44
  num_with_malformed_responses: 22
  user_asks: 23
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 9
  total_tests: 225
  command: OPENAI_API_BASE=https://dashscope-intl.aliyuncs.com/compatible-mode/v1 aider --model openai/qwen-max-2025-01-25
  date: 2025-01-28
  versions: 0.72.4.dev
  seconds_per_case: 39.5

- dirname: 2025-01-31-20-27-46--o3-mini-diff2
  test_cases: 225
  model: o3-mini (medium)
  edit_format: diff
  commit_hash: 2fb517b-dirty
  pass_rate_1: 19.1
  pass_rate_2: 53.8
  pass_num_1: 43
  pass_num_2: 121
  percent_cases_well_formed: 95.1
  error_outputs: 28
  num_malformed_responses: 28
  num_with_malformed_responses: 11
  user_asks: 17
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 2
  total_tests: 225
  command: aider --model o3-mini
  date: 2025-01-31
  versions: 0.72.4.dev
  seconds_per_case: 47.2
  total_cost: 8.8599

- dirname: 2025-01-31-20-42-47--o3-mini-diff-high
  test_cases: 224
  model: o3-mini (high)
  edit_format: diff
  commit_hash: b0d58d1-dirty
  pass_rate_1: 21.0
  pass_rate_2: 60.4
  pass_num_1: 47
  pass_num_2: 136
  percent_cases_well_formed: 93.3
  error_outputs: 26
  num_malformed_responses: 24
  num_with_malformed_responses: 15
  user_asks: 19
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 7
  total_tests: 225
  command: aider --model o3-mini --reasoning-effort high
  date: 2025-01-31
  versions: 0.72.4.dev
  seconds_per_case: 124.6
  total_cost: 18.1584

- dirname: 2025-01-21-22-51-49--gemini-2.0-flash-thinking-exp-01-21-polyglot-diff
  test_cases: 225
  model: gemini-2.0-flash-thinking-exp-01-21
  edit_format: diff
  commit_hash: 843720a
  pass_rate_1: 5.8
  pass_rate_2: 18.2
  pass_num_1: 13
  pass_num_2: 41
  percent_cases_well_formed: 77.8
  error_outputs: 182
  num_malformed_responses: 180
  num_with_malformed_responses: 50
  user_asks: 26
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 2
  test_timeouts: 7
  total_tests: 225
  command: aider --model gemini/gemini-2.0-flash-thinking-exp-01-21
  date: 2025-01-21
  versions: 0.72.2.dev
  seconds_per_case: 24.2
  total_cost: 0.0000

- dirname: 2025-02-15-19-51-22--chatgpt4o-feb15-diff
  test_cases: 223
  model: chatgpt-4o-latest (2025-02-15)
  edit_format: diff
  commit_hash: 108ce18-dirty
  pass_rate_1: 9.0
  pass_rate_2: 27.1
  pass_num_1: 20
  pass_num_2: 61
  percent_cases_well_formed: 93.3
  error_outputs: 66
  num_malformed_responses: 21
  num_with_malformed_responses: 15
  user_asks: 57
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 2
  total_tests: 225
  command: aider --model chatgpt-4o-latest
  date: 2025-02-15
  versions: 0.74.3.dev
  seconds_per_case: 12.4
  total_cost: 14.3703

- dirname: 2025-02-24-19-54-07--sonnet37-diff
  test_cases: 225
  model: claude-3-7-sonnet-20250219 (no thinking)
  edit_format: diff
  commit_hash: 75e9ee6
  pass_rate_1: 24.4
  pass_rate_2: 60.4
  pass_num_1: 55
  pass_num_2: 136
  percent_cases_well_formed: 93.3
  error_outputs: 16
  num_malformed_responses: 16
  num_with_malformed_responses: 15
  user_asks: 12
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 0
  total_tests: 225
  command: aider --model sonnet
  date: 2025-02-24
  versions: 0.74.4.dev
  seconds_per_case: 28.3
  total_cost: 17.7191

- dirname: 2025-02-24-21-47-23--sonnet37-diff-think-32k-64k
  test_cases: 225
  model: claude-3-7-sonnet-20250219 (32k thinking tokens)
  edit_format: diff
  commit_hash: 60d11a6, 93edbda
  pass_rate_1: 29.3
  pass_rate_2: 64.9
  pass_num_1: 66
  pass_num_2: 146
  percent_cases_well_formed: 97.8
  error_outputs: 66
  num_malformed_responses: 5
  num_with_malformed_responses: 5
  user_asks: 5
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 1
  total_tests: 225
  command: "aider --model anthropic/claude-3-7-sonnet-20250219 # plus yml config"
  date: 2025-02-24
  versions: 0.75.1.dev
  seconds_per_case: 105.2
  total_cost: 36.8343

- dirname: 2025-02-27-20-26-15--gpt45-diff3
  test_cases: 224
  model: gpt-4.5-preview
  edit_format: diff
  commit_hash: b462e55-dirty
  pass_rate_1: 22.3
  pass_rate_2: 44.9
  pass_num_1: 50
  pass_num_2: 101
  percent_cases_well_formed: 97.3
  error_outputs: 10
  num_malformed_responses: 8
  num_with_malformed_responses: 6
  user_asks: 15
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 2
  total_tests: 225
  command: aider --model openai/gpt-4.5-preview
  date: 2025-02-27
  versions: 0.75.2.dev
  seconds_per_case: 113.5
  total_cost: 183.1802

- dirname: 2025-03-06-17-40-24--qwq32b-diff-temp-topp-ex-sys-remind-user-for-real
  test_cases: 225
  model: QwQ-32B
  edit_format: diff
  commit_hash: 51d118f-dirty
  pass_rate_1: 8.0
  pass_rate_2: 20.9
  pass_num_1: 18
  pass_num_2: 47
  percent_cases_well_formed: 67.6
  error_outputs: 145
  num_malformed_responses: 143
  num_with_malformed_responses: 73
  user_asks: 17
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 4
  total_tests: 225
  command: aider --model fireworks_ai/accounts/fireworks/models/qwq-32b
  date: 2025-03-06
  versions: 0.75.3.dev
  seconds_per_case: 228.6
  total_cost: 0.0000

- dirname: 2025-03-07-15-11-27--qwq32b-arch-temp-topp-again
  test_cases: 225
  model: QwQ-32B + Qwen 2.5 Coder Instruct
  edit_format: architect
  commit_hash: 52162a5
  editor_model: fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct
  editor_edit_format: editor-diff
  pass_rate_1: 9.8
  pass_rate_2: 26.2
  pass_num_1: 22
  pass_num_2: 59
  percent_cases_well_formed: 100.0
  error_outputs: 122
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 489
  lazy_comments: 8
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 2
  total_tests: 225
  command: aider --model fireworks_ai/accounts/fireworks/models/qwq-32b --architect
  date: 2025-03-07
  versions: 0.75.3.dev
  seconds_per_case: 137.4
  total_cost: 0

- dirname: 2025-03-14-23-40-00--cmda-quality-whole2
  test_cases: 225
  model: command-a-03-2025-quality
  edit_format: whole
  commit_hash: a1aa63f
  pass_rate_1: 2.2
  pass_rate_2: 12.0
  pass_num_1: 5
  pass_num_2: 27
  percent_cases_well_formed: 99.6
  error_outputs: 2
  num_malformed_responses: 1
  num_with_malformed_responses: 1
  user_asks: 215
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 4
  total_tests: 225
  command: OPENAI_API_BASE=https://api.cohere.ai/compatibility/v1 aider --model openai/command-a-03-2025-quality
  date: 2025-03-14
  versions: 0.77.1.dev
  seconds_per_case: 85.1
  total_cost: 0.0000

- dirname: 2025-03-15-01-21-24--gemma3-27b-or
  test_cases: 225
  model: gemma-3-27b-it
  edit_format: whole
  commit_hash: fd21f51-dirty
  pass_rate_1: 1.8
  pass_rate_2: 4.9
  pass_num_1: 4
  pass_num_2: 11
  percent_cases_well_formed: 100.0
  error_outputs: 3
  num_malformed_responses: 0
  num_with_malformed_responses: 0
  user_asks: 181
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 1
  test_timeouts: 3
  total_tests: 225
  command: aider --model openrouter/google/gemma-3-27b-it
  date: 2025-03-15
  versions: 0.77.1.dev
  seconds_per_case: 79.7
  total_cost: 0.0000

- dirname: 2025-03-24-15-41-33--deepseek-v3-0324-polyglot-diff
  test_cases: 225
  model: DeepSeek V3 (0324)
  edit_format: diff
  commit_hash: 502b863
  pass_rate_1: 28.0
  pass_rate_2: 55.1
  pass_num_1: 63
  pass_num_2: 124
  percent_cases_well_formed: 99.6
  error_outputs: 32
  num_malformed_responses: 1
  num_with_malformed_responses: 1
  user_asks: 96
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 2
  test_timeouts: 4
  total_tests: 225
  command: aider --model deepseek/deepseek-chat
  date: 2025-03-24
  versions: 0.78.1.dev
  seconds_per_case: 290.0
  total_cost: 1.1164

- dirname: 2025-03-25-19-46-45--gemini-25-pro-exp-diff-fenced
  test_cases: 225
  model: Gemini 2.5 Pro exp-03-25
  edit_format: diff-fenced
  commit_hash: 33413ec
  pass_rate_1: 39.1
  pass_rate_2: 72.9
  pass_num_1: 88
  pass_num_2: 164
  percent_cases_well_formed: 89.8
  error_outputs: 30
  num_malformed_responses: 30
  num_with_malformed_responses: 23
  user_asks: 57
  lazy_comments: 0
  syntax_errors: 0
  indentation_errors: 0
  exhausted_context_windows: 0
  test_timeouts: 3
  total_tests: 225
  command: aider --model gemini/gemini-2.5-pro-exp-03-25
  date: 2025-03-25
  versions: 0.78.1.dev
  seconds_per_case: 47.1
  total_cost: 0.0000