-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmodels_config.yaml
More file actions
152 lines (129 loc) · 3.34 KB
/
Copy pathmodels_config.yaml
File metadata and controls
152 lines (129 loc) · 3.34 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
# Model configuration for PBF Comics Benchmark
models:
claude-3-opus:
provider: anthropic
model: claude-3-opus-20240229
api_key_env: ANTHROPIC_API_KEY
max_tokens: 1000
temperature: 0.7
claude-3-5-sonnet:
provider: anthropic
model: claude-3-5-sonnet-20241022
api_key_env: ANTHROPIC_API_KEY
max_tokens: 1000
temperature: 0.7
claude-3-7-sonnet:
provider: anthropic
model: claude-3-7-sonnet-20250219
api_key_env: ANTHROPIC_API_KEY
max_tokens: 1000
temperature: 0.7
claude-4-sonnet:
provider: anthropic
model: claude-sonnet-4-20250514
api_key_env: ANTHROPIC_API_KEY
max_tokens: 1000
temperature: 0.7
claude-4-opus:
provider: anthropic
model: claude-opus-4-20250514
api_key_env: ANTHROPIC_API_KEY
max_tokens: 1000
temperature: 0.7
claude-4.1-opus:
provider: anthropic
model: claude-opus-4-1-20250805
api_key_env: ANTHROPIC_API_KEY
max_tokens: 1000
temperature: 0.7
claude-4.5-sonnet:
provider: anthropic
model: claude-sonnet-4-5-20250929
api_key_env: ANTHROPIC_API_KEY
max_tokens: 1000
temperature: 0.7
gemini-2.0-flash:
provider: google
model: gemini-2.0-flash-exp
api_key_env: GOOGLE_API_KEY
max_tokens: 1000
temperature: 0.7
gemini-2.5-flash:
provider: google
model: gemini-2.5-flash
api_key_env: GOOGLE_API_KEY
max_tokens: 1000
temperature: 0.7
gemini-2.5-pro:
provider: google
model: gemini-2.5-pro
api_key_env: GOOGLE_API_KEY
max_tokens: 1000
temperature: 0.7
gpt-4o:
provider: openai
model: gpt-4o
api_key_env: OPENAI_API_KEY
max_tokens: 1000
temperature: 0.7
gpt-4o-mini:
provider: openai
model: gpt-4o-mini
api_key_env: OPENAI_API_KEY
max_tokens: 1000
temperature: 0.7
gpt-4.1:
provider: openai
model: gpt-4.1-2025-04-14
api_key_env: OPENAI_API_KEY
max_tokens: 1000
temperature: 0.7
gpt-5:
provider: openai
model: gpt-5-2025-08-07
api_key_env: OPENAI_API_KEY
max_tokens: 1000
temperature: 0.7
gpt-5-mini:
provider: openai
model: gpt-5-mini-2025-08-07
api_key_env: OPENAI_API_KEY
max_tokens: 1000
temperature: 0.7
o3-mini:
provider: openai
model: o3-mini-2025-01-31
api_key_env: OPENAI_API_KEY
max_tokens: 1000
temperature: 0.7
o4-mini:
provider: openai
model: o4-mini-2025-04-16
api_key_env: OPENAI_API_KEY
max_tokens: 1000
temperature: 0.7
grok-4-0709:
provider: xai
model: grok-4-0709
api_key_env: XAI_API_KEY
max_tokens: 1000
temperature: 0.7
prompts:
explain_comic: "Explain this comic. Describe what's happening and explain the humor or message."
# Models to use for generating ground truth candidates
phase1_models: [claude-3-5-sonnet, gemini-2.0-flash, gpt-4o]
# Models to evaluate in the benchmark
benchmark_models: [claude-4-sonnet, claude-4-opus, claude-4.1-opus, gemini-2.0-flash, gemini-2.5-flash, gemini-2.5-pro, gpt-4o, gpt-4o-mini, gpt-4.1, gpt-5, gpt-5-mini, o3-mini, o4-mini, grok-4-0709]
# Model to use as judge
judge_model: claude-4.1-opus
# Rate limiting settings (requests per minute)
rate_limits:
anthropic: 50
google: 60
openai: 60
xai: 60
# Retry settings
retry:
max_attempts: 3
initial_delay: 1
backoff_factor: 2