Repository navigation
Expand file tree
/
Copy pathmain.py
More file actions
639 lines (594 loc) · 27.9 KB
/
Copy pathmain.py
File metadata and controls
639 lines (594 loc) · 27.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
import argparse
import json
import logging
from pathlib import Path
import sys
from utils.common import (
ASYNC_LOOP,
extract_xml_content,
strip_think_simple,
get_current_log_path,
_compute_binary_metrics,
save_progressive_iteration_samples,
prepare_dataset,
NaiveProver
)
from utils.verifiers import (
Verifier,
PessimisticVerifier,
VPessimisticVerifier,
ProgressivePessimisticVerifier,
PessimisticPruningVerifier,
NoneVerifier,
)
from utils.prover_pipeline import ProverPipeline
from utils.arxiv_bench import (
ARXIV_MATH_GRADING_BENCH,
DEFAULT_ARXIV_DATA_DIR,
ErrorLocationJudge,
compute_iteration_location_metrics,
compute_location_tnr,
)
def main():
# Define common arguments in a parent parser
common_parser = argparse.ArgumentParser(add_help=False)
# Global/Common arguments
common_parser.add_argument("-ed", "--eval_dataset", help="the path to the dataset used for evaluation", default="")
common_parser.add_argument("-pm", "--proof_model", help="model that generates proofs for given problems", default="")
common_parser.add_argument("-em", "--eval_model", help="the model used for evaluation (if needed)", default="")
common_parser.add_argument("--log_dir", help="the logging directory path", default="eval_logs")
common_parser.add_argument("--reasoning_effort", help="the reasoning_effort parameter for some models", default="medium", choices=["minimal", "low", "medium", "high"])
common_parser.add_argument("--concurrency", type=int, default=8, help="maximum number of concurrent API requests")
common_parser.add_argument("--reviewer", default="standard", choices=["standard", "pessimistic", "vpessimistic", "progressive", "ppruning", "none"], help="the reviewer used for evaluation")
common_parser.add_argument("--reviews", type=int, default=3, help="maximum reviews per sample for multi-review verifiers (pessimistic/ppruning)")
common_parser.add_argument("--chunk_length", type=int, default=7, help="lines per chunk for vpessimistic reviewer")
common_parser.add_argument("--progressive_max_iters", type=int, default=3, help="maximum refinement passes for progressive reviewer")
common_parser.add_argument("--progressive_min_chunk_size", type=int, default=6, help="minimum lines per chunk for progressive reviewer")
common_parser.add_argument("--prover_base_url", default="", help="the base url for prover")
common_parser.add_argument("--eval_base_url", default="", help="the base url for evaluator")
common_parser.add_argument("--prover_api_key", default="", help="the api key for the prover")
common_parser.add_argument("--eval_api_key", default="", help="the api key for the evaluator")
common_parser.add_argument("--enable_thinking", action=argparse.BooleanOptionalAction, default=True, help="toggle enable_thinking parameter for models that support reasoning traces")
common_parser.add_argument(
"--arxiv_data_dir",
default=str(DEFAULT_ARXIV_DATA_DIR),
help="prepared ArxivMathGradingBench directory containing manifest.jsonl",
)
common_parser.add_argument(
"--location_judge_model",
default="",
help="independent error-location judge model (defaults to --eval_model)",
)
common_parser.add_argument(
"--location_judge_base_url",
default="",
help="location judge endpoint (defaults to evaluator endpoint)",
)
common_parser.add_argument(
"--location_judge_api_key",
default="",
help="location judge API key (defaults to evaluator key)",
)
common_parser.add_argument(
"--verifier_samples",
default="",
help=(
"path to a previously generated verifier_samples.json. "
"When set, uses the same problems, proofs, and golden labels from the file, "
"skipping new proof generation and ground-truth verification."
),
)
# Main parser inherits from common_parser
parser = argparse.ArgumentParser(
description="RProver",
parents=[common_parser]
)
# Subcommands
subparsers = parser.add_subparsers(dest="command", help="sub-command help")
# 'prove' subcommand inherits from common_parser
prove_parser = subparsers.add_parser("prove", help="Run the iterative prover pipeline", parents=[common_parser])
prove_parser.add_argument("--dataset", required=True, help="Dataset to use (overrides -ed if present)")
prove_parser.add_argument("--prover", required=True, help="Prover model name (overrides -pm if present)")
prove_parser.add_argument("--refine_iters", type=int, default=3, help="Maximum number of refinement iterations")
prove_parser.add_argument("--resume", help="Path to a previous log directory to resume from")
# Note: 'prove' will also use global args like --reviewer, --reviews, etc.
logger = logging.getLogger("main")
args = parser.parse_args()
# Handle 'prove' subcommand
if args.command == "prove":
# Override global args with subcommand specific args if needed/provided
# But here we used unique names mostly. Let's sync them for clarity
dataset_path = args.dataset
prover_model = args.prover
# Set args for compatibility with functions that use them
args.eval_dataset = dataset_path
args.proof_model = prover_model
logger.info(f"Starting 'prove' pipeline on {dataset_path} with prover {prover_model}")
# Resolve API bases
prover_base_url = args.prover_base_url
prover_api_key = args.prover_api_key
eval_base_url = args.eval_base_url or prover_base_url
eval_api_key = args.eval_api_key or prover_api_key
# Instantiate Pipeline
pipeline = ProverPipeline(
args=args,
dataset_path=dataset_path,
prover_model=prover_model,
prover_api_base=prover_base_url,
prover_api_key=prover_api_key,
reviewer_model=args.eval_model or prover_model, # Default to prover if eval not set? or check if empty.
reviewer_api_base=eval_base_url,
reviewer_api_key=eval_api_key,
max_refine_iters=args.refine_iters,
resume_path=args.resume
)
ASYNC_LOOP.run(pipeline.run())
return
# --- Standard 'main' logic (default behavior) ---
logger.info("start verifying with proof_model: %s", args.proof_model)
logger.info("using eval model: %s", args.eval_model)
# If verifier_samples is provided, use it to load problems/proofs and GT labels
loaded_verifier_samples = None
preloaded_gt_labels = None
preloaded_gt_texts = None
dataset_metadata = []
is_arxiv_bench = (
args.verifier_samples == ARXIV_MATH_GRADING_BENCH
or (
not args.verifier_samples
and args.eval_dataset == ARXIV_MATH_GRADING_BENCH
)
)
if args.verifier_samples or is_arxiv_bench:
if is_arxiv_bench:
ds = prepare_dataset(
ARXIV_MATH_GRADING_BENCH,
arxiv_data_dir=args.arxiv_data_dir,
)
problems = ds["problem"]
proofs = ds["proof"]
preloaded_gt_labels = ds["gt_eval"]
preloaded_gt_texts = ds["gt_error_location"]
dataset_metadata = [
{
"arxiv_id": row.get("arxiv_id"),
"version": row.get("version"),
"title": row.get("title_extracted_from_tex"),
"ground_truth_error_location": row.get("gt_error_location"),
"pdf_path": row.get("pdf_path"),
"source_archive_path": row.get("source_archive_path"),
"proof_path": row.get("proof_path"),
}
for row in ds
]
logger.info(
"Loaded %d complete papers from prepared %s data",
len(problems),
ARXIV_MATH_GRADING_BENCH,
)
elif args.verifier_samples == "Salesforce/Hard2Verify" or args.verifier_samples == "NP_dataset/gradingbench.csv":
ds = prepare_dataset(args.verifier_samples)
problems = ds["problem"]
proofs = ds["proof"]
# preloaded_gt_texts = [e["human_labels_first_error_idx"] for e in ds]
preloaded_gt_labels = ds["gt_eval"]
preloaded_gt_texts = [None] * len(problems)
logger.info("Loaded %d samples from verifier_samples: %s", len(problems), args.verifier_samples)
else:
vs_path = Path(args.verifier_samples) / "samples.json"
with vs_path.open("r", encoding="utf-8") as f:
loaded_verifier_samples = json.load(f)
if not isinstance(loaded_verifier_samples, list):
raise ValueError("verifier_samples must be a list of sample dicts")
problems = [s.get("problem", "") for s in loaded_verifier_samples]
proofs = [s.get("proof", "") for s in loaded_verifier_samples]
preloaded_gt_labels = [bool(s.get("gt_label", False)) for s in loaded_verifier_samples]
preloaded_gt_texts = [s.get("verification", "") for s in loaded_verifier_samples]
logger.info("Loaded %d samples from verifier_samples: %s", len(problems), vs_path)
else:
ds = prepare_dataset(args.eval_dataset)
problems = [e['problem'] for e in ds]
# Resolve API bases and keys with fallback: prover -> eval
prover_base_url = args.prover_base_url
prover_api_key = args.prover_api_key
eval_base_url = args.eval_base_url or prover_base_url
eval_api_key = args.eval_api_key or prover_api_key
prover = None
logdir = get_current_log_path(args.log_dir)
logdir.mkdir(parents=True, exist_ok=True)
# Collect proofs unless verifier_samples is provided
if args.verifier_samples or is_arxiv_bench:
striped_proofs = (
list(proofs)
if is_arxiv_bench
else [strip_think_simple(proof) for proof in proofs]
)
logger.info(
"Using preloaded proofs; prover construction and generation are skipped"
)
else:
prover = NaiveProver(
api_base=prover_base_url,
api_key=prover_api_key,
model=args.proof_model,
)
proofs = prover(
problems,
reasoning_effort=args.reasoning_effort,
enable_thinking=args.enable_thinking,
concurrency=args.concurrency,
)
striped_proofs = [strip_think_simple(proof) for proof in proofs]
logger.info("successfully collected %d proofs from %s", len(proofs), args.proof_model)
if args.reviewer == "pessimistic":
# Use the new PessimisticVerifier (first error wins, also records majority vote)
evaluator = PessimisticVerifier(eval_base_url, eval_api_key, args.eval_model, review_times=args.reviews)
elif args.reviewer == "vpessimistic":
# Chunked pessimistic verifier (focus per-chunk)
evaluator = VPessimisticVerifier(eval_base_url, eval_api_key, args.eval_model, chunk_length=args.chunk_length)
elif args.reviewer == "progressive":
# Progressive chunking: coarse-to-fine pessimistic verifier
evaluator = ProgressivePessimisticVerifier(
eval_base_url,
eval_api_key,
args.eval_model,
max_iters=args.progressive_max_iters,
min_chunk_size=args.progressive_min_chunk_size,
)
elif args.reviewer == "ppruning":
evaluator = PessimisticPruningVerifier(
eval_base_url,
eval_api_key,
args.eval_model,
review_times=args.reviews,
)
elif args.reviewer == "none":
evaluator = NoneVerifier(eval_base_url, eval_api_key, args.eval_model)
else:
evaluator = Verifier(eval_base_url, eval_api_key, args.eval_model)
eval_call_kwargs = {
"reasoning_effort": args.reasoning_effort,
"enable_thinking": args.enable_thinking,
"concurrency": args.concurrency,
}
if args.reviewer in {"pessimistic", "ppruning"}:
eval_call_kwargs["ground_truth_labels"] = preloaded_gt_labels
evals, verifications = evaluator(
problems,
striped_proofs,
**eval_call_kwargs,
)
if is_arxiv_bench:
request_audit = getattr(evaluator, "last_request_audit", None)
if request_audit is not None:
with (logdir / "reviewer_input_audit.json").open(
"w", encoding="utf-8"
) as handle:
json.dump(request_audit, handle, ensure_ascii=False, indent=2)
logger.info(
"Saved %d reviewer input audit records confirming whole-paper payloads",
len(request_audit),
)
accuracy = sum(evals) / len(evals) if evals else 0.0
logger.info(f"Obtained final accuracy: {accuracy}")
majority_evals = None
majority_verifications = None
majority_accuracy = None
pess_family = args.reviewer in {"pessimistic", "ppruning"}
if pess_family:
majority_results = getattr(evaluator, "last_majority_results", None)
if majority_results:
majority_evals, majority_verifications = majority_results
if len(majority_evals) == len(evals) and len(evals) > 0:
majority_accuracy = sum(majority_evals) / len(majority_evals)
logger.info(f"Majority voting accuracy from the same reviews: {majority_accuracy}")
else:
majority_evals = None
majority_verifications = None
if args.reviewer == "pessimistic":
step_logs = getattr(evaluator, "stepwise_review_logs", None)
if step_logs:
step_log_path = logdir / "pessimistic_step_metrics.json"
with step_log_path.open("w", encoding="utf-8") as f:
json.dump(step_logs, f, ensure_ascii=False, indent=2, default=str)
logger.info("Saved stepwise pessimistic review metrics to %s", step_log_path)
majority_step_logs = getattr(evaluator, "majority_step_logs", None)
majority_metrics = None
if (
preloaded_gt_labels
and majority_evals
and len(majority_evals) == len(preloaded_gt_labels)
):
gt_vector = [1 if bool(x) else 0 for x in preloaded_gt_labels]
majority_preds = [1 if bool(x) else 0 for x in majority_evals]
majority_metrics = _compute_binary_metrics(majority_preds, gt_vector)
if majority_step_logs or majority_metrics:
majority_log_path = logdir / "pessimistic_majority_metrics.json"
payload = {}
if majority_step_logs:
payload["steps"] = majority_step_logs
if majority_metrics:
payload["metrics"] = majority_metrics
with majority_log_path.open("w", encoding="utf-8") as f:
json.dump(payload, f, ensure_ascii=False, indent=2, default=str)
logger.info("Saved pessimistic majority metrics to %s", majority_log_path)
logger.info("Evaluating reviewer against ground truth")
if args.verifier_samples or is_arxiv_bench:
# Use ground-truth labels/texts from the provided verifier_samples file
gt_labels = preloaded_gt_labels
gt_texts = preloaded_gt_texts
logger.info("Using GT labels from verifier_samples; skipping new GT verification")
else:
gt_labels = [1] * len(evals)
gt_texts = [None] * len(evals)
preds = [int(x) for x in evals]
gts = [int(x) for x in gt_labels]
verifier_eval = _compute_binary_metrics(preds, gts)
location_judgments = None
if is_arxiv_bench:
location_judge = ErrorLocationJudge(
api_base=args.location_judge_base_url or eval_base_url,
api_key=args.location_judge_api_key or eval_api_key,
model=args.location_judge_model or args.eval_model,
)
location_judgments = location_judge(
striped_proofs,
preds,
verifications,
list(preloaded_gt_texts),
reasoning_effort=args.reasoning_effort,
enable_thinking=args.enable_thinking,
concurrency=args.concurrency,
)
location_matches = [entry["matched"] for entry in location_judgments]
verifier_eval["location_aware"] = compute_location_tnr(
preds,
location_matches,
)
verifier_eval["location_aware"]["judge_calls"] = sum(
1 for entry in location_judgments if entry["ran"]
)
verifier_eval["location_aware"]["judge_parse_failures"] = sum(
1
for entry in location_judgments
if entry["ran"] and not entry.get("parse_ok", False)
)
verifier_eval["location_judge_model"] = (
args.location_judge_model or args.eval_model
)
logger.info(
"ArxivMathGradingBench location-aware TNR: %.4f",
verifier_eval["location_aware"]["tnr"] or 0.0,
)
if pess_family and majority_evals and len(majority_evals) == len(preds):
majority_preds = [int(x) for x in majority_evals]
verifier_eval["majority_vote_metrics"] = _compute_binary_metrics(majority_preds, gts)
pred_history = getattr(evaluator, "iteration_prediction_history", [])
# Save sample-level comparison
majority_sample_fields = None
if pred_history:
resolved_history = getattr(evaluator, "iteration_resolved_predictions", [])
pending_masks = getattr(evaluator, "iteration_pending_masks", [])
review_costs = getattr(evaluator, "iteration_review_costs", [])
iteration_location_metrics = {}
if is_arxiv_bench and location_judgments is not None:
iteration_location_metrics = {
entry["iteration_index"]: entry["metrics"]
for entry in compute_iteration_location_metrics(
pred_history,
location_judgments,
)
}
iteration_metrics = []
total_samples = len(gts)
for idx, preds_if_stop in enumerate(pred_history):
iteration_index = idx + 1
metrics_if_stop = _compute_binary_metrics(preds_if_stop, gts)
resolved_preds = resolved_history[idx] if idx < len(resolved_history) else [None] * total_samples
resolved_metrics = _compute_binary_metrics(resolved_preds, gts)
pending_mask = pending_masks[idx] if idx < len(pending_masks) else [False] * total_samples
pending_samples = sum(1 for flag in pending_mask if flag)
cost_info = review_costs[idx] if idx < len(review_costs) else {}
iteration_entry = {
"iteration_index": iteration_index,
"metrics_if_stopped": metrics_if_stop,
"resolved_metrics": resolved_metrics,
"pending_samples": pending_samples,
"resolved_samples": total_samples - pending_samples,
"reviews_this_iter": cost_info.get("reviews_this_iter"),
"cumulative_reviews": cost_info.get("cumulative_reviews"),
"avg_input_tokens_this_iter": cost_info.get("avg_input_tokens_this_iter"),
"avg_output_tokens_this_iter": cost_info.get("avg_output_tokens_this_iter"),
"avg_input_tokens_cumulative": cost_info.get("avg_input_tokens_cumulative"),
"avg_output_tokens_cumulative": cost_info.get("avg_output_tokens_cumulative"),
}
if iteration_index in iteration_location_metrics:
iteration_entry["location_aware_metrics"] = (
iteration_location_metrics[iteration_index]
)
iteration_metrics.append(iteration_entry)
if iteration_metrics:
key_name = "progressive_iteration_metrics" if args.reviewer == "progressive" else f"{args.reviewer}_iteration_metrics"
verifier_eval[key_name] = iteration_metrics
if (
pess_family
and majority_evals
and majority_verifications
and len(majority_evals) == len(gts)
and len(majority_verifications) == len(gts)
):
majority_sample_fields = list(zip(
[bool(int(x)) for x in majority_evals],
majority_verifications,
))
verifier_samples = []
for idx, (problem, proof, pred, pred_text, gt, gt_text) in enumerate(
zip(problems, proofs, preds, verifications, gts, gt_texts)
):
sample_entry = {
"problem": problem,
"proof": proof,
"pred_label": bool(pred),
"pred_text": pred_text,
"gt_label": bool(gt),
"gt_text": gt_text,
}
if dataset_metadata:
sample_entry["dataset_metadata"] = dataset_metadata[idx]
if location_judgments:
sample_entry["location_judgment"] = location_judgments[idx]
if majority_sample_fields:
maj_label, maj_text = majority_sample_fields[idx]
sample_entry["majority_pred_label"] = maj_label
sample_entry["majority_pred_text"] = maj_text
verifier_samples.append(sample_entry)
with open(logdir / "verifier_eval.json", "w", encoding="utf-8") as f:
json.dump(verifier_eval, f, ensure_ascii=False, indent=2, default=str)
with open(logdir / "verifier_samples.json", "w", encoding="utf-8") as f:
json.dump(verifier_samples, f, ensure_ascii=False, indent=2, default=str)
# Add summary to logs.json payload
vars_dict_key = "verifier_evaluation"
# vars_dict defined below; collect into a temporary dict for later merge
extra_verifier_eval = {vars_dict_key: verifier_eval}
logger.info("evaluation ended")
vars_dict = vars(args)
vars_dict["accuracy"] = accuracy
if is_arxiv_bench:
vars_dict["final_performance_metric"] = {
"name": "location_aware_tnr",
"value": verifier_eval["location_aware"]["tnr"],
}
if majority_accuracy is not None:
vars_dict["majority_accuracy"] = majority_accuracy
# Reviewer cost metrics for post-hoc cost/performance analysis
reviewer_cost = {"reviewer": args.reviewer}
num_samples = len(problems)
if args.reviewer in {"vpessimistic", "progressive", "ppruning"}:
if args.reviewer == "vpessimistic":
counts = getattr(evaluator, "last_chunk_counts", []) or []
else:
counts = getattr(evaluator, "last_review_counts", []) or []
total_reviews = sum(counts)
avg_per_sample = (total_reviews / len(counts)) if counts else 0.0
reviewer_cost.update({
"total_reviews": total_reviews,
"avg_reviews_per_sample": avg_per_sample,
"min_reviews_per_sample": (min(counts) if counts else 0),
"max_reviews_per_sample": (max(counts) if counts else 0),
})
else:
if args.reviewer == "standard":
per_sample = 1
elif args.reviewer == "pessimistic":
per_sample = int(args.reviews)
else:
per_sample = 1
reviewer_cost.update({
"reviews_per_sample": per_sample,
"total_reviews": per_sample * num_samples,
})
vars_dict["reviewer_cost"] = reviewer_cost
# Token stats: skip prover token stats when using preloaded samples
if args.verifier_samples or is_arxiv_bench:
average_prover_inp_tokens = None
average_prover_opt_tokens = None
else:
average_prover_inp_tokens = (
sum(prover.client.input_tokens) / len(prover.client.input_tokens)
if prover.client.input_tokens else 0.0
)
average_prover_opt_tokens = (
sum(prover.client.comp_tokens) / len(prover.client.comp_tokens)
if prover.client.comp_tokens else 0.0
)
average_eval_inp_tokens = (
sum(evaluator.client.input_tokens) / len(evaluator.client.input_tokens)
if evaluator.client.input_tokens else 0.0
)
average_eval_opt_tokens = (
sum(evaluator.client.comp_tokens) / len(evaluator.client.comp_tokens)
if evaluator.client.comp_tokens else 0.0
)
logger.info(f"Average token inputs in prover: {average_prover_inp_tokens}")
logger.info(f"Average completion tokens in prover: {average_prover_opt_tokens}")
logger.info(f"Average token inputs in evaluator: {average_eval_inp_tokens}")
logger.info(f"Average completion inputs in evaluator: {average_eval_opt_tokens}")
vars_dict["average_prover_inp_tokens"] = average_prover_inp_tokens
vars_dict["average_prover_opt_tokens"] = average_prover_opt_tokens
vars_dict["average_eval_inp_tokens"] = average_eval_inp_tokens
vars_dict["average_eval_opt_tokens"] = average_eval_opt_tokens
if location_judgments:
vars_dict["location_judge_model"] = args.location_judge_model or args.eval_model
vars_dict["location_judge_average_input_tokens"] = (
sum(location_judge.client.input_tokens) / len(location_judge.client.input_tokens)
if location_judge.client.input_tokens else 0.0
)
vars_dict["location_judge_average_output_tokens"] = (
sum(location_judge.client.comp_tokens) / len(location_judge.client.comp_tokens)
if location_judge.client.comp_tokens else 0.0
)
# Merge verifier evaluation summary if available
try:
if 'extra_verifier_eval' in locals():
vars_dict.update(extra_verifier_eval)
except Exception:
pass
with open(logdir / "logs.json", "w", encoding="utf-8") as f:
json.dump(vars_dict, f, ensure_ascii=False, indent=2, default=str)
# Prepare sample payload; use placeholder tokens if verifier_samples is provided
if args.verifier_samples or is_arxiv_bench:
prover_inp_tokens = [None] * len(problems)
prover_comp_tokens = [None] * len(problems)
else:
prover_inp_tokens = prover.client.last_input_tokens
prover_comp_tokens = prover.client.last_comp_tokens
samples = [
{
"problem": problem,
"proof": proof,
"eval": eval,
"verification": verification,
"input_tokens": inp_tokens,
"completion_tokens": comp_tokens
}
for (problem, proof, eval, verification, inp_tokens, comp_tokens) in zip(
problems, proofs, evals, verifications, prover_inp_tokens, prover_comp_tokens
)
]
if majority_evals and majority_verifications and len(samples) == len(majority_evals):
for sample, maj_eval, maj_verification in zip(samples, majority_evals, majority_verifications):
sample["majority_eval"] = maj_eval
sample["majority_verification"] = maj_verification
if dataset_metadata:
for sample, metadata in zip(samples, dataset_metadata):
sample["dataset_metadata"] = metadata
if location_judgments:
for sample, judgment in zip(samples, location_judgments):
sample["location_judgment"] = judgment
with open(logdir / "samples.json", "w", encoding="utf-8") as f:
json.dump(samples, f, ensure_ascii=False, indent=2, default=str)
if args.reviewer in {"progressive", "ppruning"}:
prefix = "progressive" if args.reviewer == "progressive" else "ppruning"
save_progressive_iteration_samples(
logdir,
getattr(evaluator, "iteration_samples_log", []),
problems,
proofs,
striped_proofs,
getattr(evaluator, "iteration_summary", []),
getattr(evaluator, "iteration_review_costs", []),
prefix=prefix,
)
logger.info(f"successfully saved logs to path {logdir}")
if __name__ == "__main__":
LOG_FMT = "%(asctime)s | %(levelname)s | %(name)s | %(message)s"
DATE_FMT = "%Y-%m-%d %H:%M:%S"
logging.basicConfig(
level=logging.INFO,
format=LOG_FMT,
datefmt=DATE_FMT,
force=True
)
logger = logging.getLogger(__name__)
logger.info("Program Started")
main()