Views
No views yet
ac1. Checkpoint saved
after training step 19 (0-indexed). Strict upstream eval parity:
1100s hard kill, verbatim prompts/entrypoints, group 64x8, T=1.0, kl 0.1.1{
2 "step": 19,
3 "progress/batch": 19,
4 "optim/lr": 4e-05,
5 "progress/done_frac": 0.4,
6 "puct/buffer_size": 312,
7 "puct/sampled_size": 8,
8 "puct/T": 9728,
9 "puct/scale_last": 0.39927465088421665,
10 "puct/buffer_value/mean": -1.5274040777183318,
11 "puct/buffer_value/std": 0.08805290550563966,
12 "puct/buffer_value/min": -2.000000000000008,
13 "puct/buffer_value/max": -1.5063483899098538,
14 "puct/buffer_timestep/mean": 8.743589743589743,
15 "puct/buffer_timestep/std": 5.632861224571347,
16 "puct/buffer_timestep/min": -1.0,
17 "puct/buffer_timestep/max": 18.0,
18 "puct/buffer_construction_len/mean": 1139.5224358974358,
19 "puct/buffer_construction_len/std": 753.9170358418278,
20 "puct/buffer_construction_len/min": 1000.0,
21 "puct/buffer_construction_len/max": 7397.0,
22 "puct/sampled_value/mean": -1.5063638011490244,
23 "puct/sampled_value/std": 2.6986580224077994e-06,
24 "puct/sampled_value/min": -1.5063678682407644,
25 "puct/sampled_value/max": -1.5063602760780042,
26 "puct/sampled_timestep/mean": 18.0,
27 "puct/sampled_timestep/std": 0.0,
28 "puct/sampled_timestep/min": 18.0,
29 "puct/sampled_timestep/max": 18.0,
30 "puct/sampled_construction_len/mean": 1000.0,
31 "puct/sampled_construction_len/std": 0.0,
32 "puct/sampled_construction_len/min": 1000.0,
33 "puct/sampled_construction_len/max": 1000.0,
34 "time/sampling": 4608.2474319934845,
35 "env/all/ac_tokens_per_turn": 9406.84375,
36 "env/all/ob_tokens_per_turn": 2877.75,
37 "env/all/turns_per_episode": 1.0,
38 "env/all/total_episodes": 512,
39 "env/all/total_turns": 512,
40 "env/all/total_ac_tokens": 4816304,
41 "env/all/total_ob_tokens": 1473408,
42 "env/all/time/sampling_mean": 347.8715773243457,
43 "env/all/time/sampling_max": 482.70458579063416,
44 "env/all/time/env_step_mean": 2119.0660397135653,
45 "env/all/time/env_step_max": 4154.340970277786,
46 "env/all/reward/mean": 0.5856794215018918,
47 "env/all/reward/max": 0.6638518083901591,
48 "env/all/reward/min": 0.0,
49 "env/all/format": 1.0,
50 "env/all/format/min": 1.0,
51 "env/all/format/max": 1.0,
52 "env/all/reward": 0.5856794215018918,
53 "env/all/correctness": 0.939453125,
54 "env/all/correctness/min": 0.0,
55 "env/all/correctness/max": 1.0,
56 "env/all/raw_score": 5.877981925929509,
57 "env/all/raw_score/min": 1.5063602760780035,
58 "env/all/raw_score/max": 666.6666666666666,
59 "env/all/initial_raw_score": -1.5063638011490244,
60 "env/all/initial_raw_score/min": -1.5063678682407644,
61 "env/all/initial_raw_score/max": -1.5063602760780042,
62 "env/all/msg": "Success; raw_score=1.5063602760780042",
63 "env/all/parsed_code": "```python\nimport time\nimport numpy as np\nfrom scipy.optimize import linprog\nimport random\nimport copy\n\ndef propose_candidate(seed=42, budget_s=1000, **kwargs):\n \"\"\"\n Proposes a sequence of non-negative numbers to minimize the evaluation score \n using a multi-stage optimization strategy with adaptive exploration, enhanced global perturbation, \n and improved LP solver settings.\n \n This version increases exploration, improves LP performance, and adds dynamic strategy adjustments to \n find better sequences than the previous approach.\n \"\"\"\n np.random.seed(seed)\n deadline = time.time() + budget_s - 10\n\n # Initialize with a diverse starting point using multiple evenly spread peaks\n n = 1000\n best_sequence = None\n if 'height_sequence_1' in globals() and isinstance(height_sequence_1, (list, np.ndarray)):\n best_sequence = list(height_sequence_1)\n else:\n # Start with a structured sequence with 25 evenly spread peaks\n total_mass = np.sqrt(2 * n)\n peak_positions = [int(i * n / 25) for i in range(25)]\n seq = np.zeros(n)\n for idx in peak_positions:\n seq[idx] = total_mass / 25\n best_sequence = seq.tolist()\n \n # Normalize the initial sequence\n initial_sum = np.sum(best_sequence)\n if initial_sum > 0:\n best_sequence = [x / initial_sum * np.sqrt(2 * n) for x in best_sequence]\n \n curr_sequence = [float(x) for x in best_sequence]\n best_score = evaluate_sequence(curr_sequence)\n\n # Parameters for optimization with enhanced exploration\n perturb_rate = 0.9 # Larger initial perturbation rate\n blend_factor = 0.95 # High blend factor to encourage LP influence\n min_perturb = 0.01\n no_improvement_threshold = 3 # Frequent perturbations\n perturb_decay_factor = 0.95 # Slower decay to encourage exploration\n global_perturb_interval = 5 # More frequent global perturbations\n global_perturb_rate = 0.4 # Higher perturbation rate for global steps\n blend_decay_factor = 0.95 # Reduced decay to maintain impact over time\n explore_rate = 0.5 # Increased exploration rate\n\n def solve_convolution_lp(f_sequence, rhs, n):\n \"\"\"Solves LP to maximize sum(b) under convolution constraint\"\"\"\n if n == 0:\n return None\n\n # Build the objective function (maximize sum(b) = minimize -sum(b))\n c = -np.ones(n)\n\n # Construct the constraints: (f * b)[k] <= rhs for all k\n A_ub = []\n b_ub = []\n\n for k in range(2 * n - 1):\n row = np.zeros(n)\n for i in range(n):\n j = k - i\n if 0 <= j < n:\n row[j] = f_sequence[i]\n A_ub.append(row)\n b_ub.append(rhs)\n\n # Non-negativity constraints: b >= 0\n a_ub_nonneg = -np.eye(n)\n b_ub_nonneg = np.zeros(n)\n A_ub = np.vstack([A_ub, a_ub_nonneg])\n b_ub = np.hstack([b_ub, b_ub_nonneg])\n\n try:\n result = linprog(\n c,\n A_ub=A_ub,\n b_ub=b_ub,\n bounds=(0.0, 1000.0),\n method='highs',\n options={\n \"time_limit\": 180.0, # Longer time for LP\n \"disp\": False,\n },\n )\n if result.success:\n return result.x\n except:\n pass\n return None\n\n def perturb_sequence(sequence, idxs, perturb_amount):\n \"\"\"Perturb multiple indices in sequence with dynamic scaling.\"\"\"\n new_seq = sequence.copy()\n for idx in idxs:\n # Apply perturbation with dynamic scaling\n delta = perturb_amount * new_seq[idx]\n new_seq[idx] = max(0.0, new_seq[idx] - delta)\n new_seq[idx] = min(1000.0, new_seq[idx])\n return new_seq\n\n iteration_count = 0\n while time.time() < deadline:\n try:\n # Compute current convolution and max_b\n conv = np.convolve(curr_sequence, curr_sequence)\n max_b = np.max(conv)\n sum_a = np.sum(curr_sequence)\n if sum_a < 0.01:\n raise ValueError(\"Sum too small\")\n\n # Normalize for LP\n normalized_seq = curr_sequence\n rhs = max_b\n\n # Solve LP to maximize sum(b) under convolution constraint\n g_fun = solve_convolution_lp(normalized_seq, rhs, n)\n if g_fun is not None and np.sum(g_fun) > 0.0:\n sum_g = np.sum(g_fun)\n # Scale the LP solution to match the original normalized sequence's normalization\n normalized_g = [x / sum_g * np.sqrt(2 * n) for x in g_fun]\n\n # Blend original and LP solution with larger blend factor\n new_seq = [(1 - blend_factor) * x + blend_factor * y for x, y in zip(curr_sequence, normalized_g)]\n curr_sequence = new_seq\n # Adjust blend factor over time\n blend_factor = max(0.6, blend_factor * blend_decay_factor)\n\n else:\n # Fallback to targeted perturbation with increased exploration\n if no_improvement_counter >= no_improvement_threshold:\n perturb_rate = min(0.9, perturb_rate * perturb_decay_factor)\n no_improvement_counter = 0\n else:\n perturb_rate = max(0.05, perturb_rate)\n\n # Randomly choose indices for perturbation\n conv = np.convolve(curr_sequence, curr_sequence)\n max_idx = np.argmax(conv)\n contributions = []\n for i in range(n):\n j = max_idx - i\n if 0 <= j < n:\n contributions.append((i, abs(curr_sequence[i] * curr_sequence[j])))\n if contributions:\n sorted_contributions = sorted(contributions, key=lambda x: x[1], reverse=True)\n idxs = [sorted_contributions[i][0] for i in range(min(5, len(sorted_contributions)))]\n\n # Also add some random indices and perturb all\n random_indices = np.random.choice(n, size=5, replace=False)\n idxs = list(set(idxs + random_indices)) # to avoid duplicates\n\n curr_sequence = perturb_sequence(curr_sequence, idxs, perturb_rate)\n else:\n idx = np.random.randint(0, n)\n curr_sequence = perturb_sequence(curr_sequence, [idx], perturb_rate)\n\n except Exception as e:\n print(f\"Error during optimization: {e}. Skipping update.\")\n continue\n\n try:\n curr_score = evaluate_sequence(curr_sequence)\n if curr_score < best_score:\n best_score = curr_score\n best_sequence = curr_sequence.copy()\n no_improvement_counter = 0\n print(f\"New best score: {best_score}\")\n else:\n no_improvement_counter += 1\n iteration_count += 1\n except Exception:\n print(\"Evaluation error, skipping update\")\n\n return [float(max(0.0, x)) for x in best_sequence]\n```",
64 "env/all/time/policy": 347.8715773243457,
65 "env/all/time/policy/min": 175.0078148841858,
66 "env/all/time/policy/max": 482.70458579063416,
67 "env/all/time/env_step": 2119.0660397135653,
68 "env/all/time/env_step/min": 0.012063980102539062,
69 "env/all/time/env_step/max": 4154.340970277786,
70 "env/all/time/reward_compute": 5.727633833885193e-07,
71 "env/all/time/reward_compute/min": 2.421438694000244e-07,
72 "env/all/time/reward_compute/max": 1.4491379261016846e-06,
73 "env/all/by_group/frac_mixed": 1.0,
74 "env/all/by_group/frac_all_good": 0.0,
75 "env/all/by_group/frac_all_bad": 0.0,
76 "advantage/mean": 0.008993344381451607,
77 "advantage/min": -1.0,
78 "advantage/max": 1.2063794136047363,
79 "time/assemble_training_data": 10.754365682601929,
80 "time/kl_vs_base": 99.84562945365906,
81 "kl_policy_base": 0.0007736113620921969,
82 "time/train": 594.2190365791321,
83 "time/save_checkpoint": 35.661667585372925,
84 "time/total": 5351.3277242183685
85}[2026-07-09T06:24:18+00:00] job=1812628 node=node-12 ngpu=3 ntrain=1 replicas=2 flash_attn=no
[2026-07-09T08:30:32+00:00] job=1813127 node=node-22 ngpu=3 ntrain=1 replicas=2 flash_attn=yes
[2026-07-09T13:07:08+00:00] job=1813128 node=node-6 ngpu=6 ntrain=2 replicas=4 flash_attn=yes