Views
No views yet
ac2. Checkpoint saved
after training step 27 (0-indexed). Strict upstream eval parity:
1100s hard kill, verbatim prompts/entrypoints, group 64x8, T=1.0, kl 0.1.1{
2 "step": 27,
3 "progress/batch": 27,
4 "optim/lr": 4e-05,
5 "progress/done_frac": 0.56,
6 "puct/buffer_size": 437,
7 "puct/sampled_size": 8,
8 "puct/T": 13824,
9 "puct/scale_last": 0.2809843275205355,
10 "puct/buffer_value/mean": 0.9323982651440982,
11 "puct/buffer_value/std": 0.04345947621754116,
12 "puct/buffer_value/min": 0.6666666666666636,
13 "puct/buffer_value/max": 0.9476509941872021,
14 "puct/buffer_timestep/mean": 12.812356979405035,
15 "puct/buffer_timestep/std": 7.926235045476226,
16 "puct/buffer_timestep/min": -1.0,
17 "puct/buffer_timestep/max": 26.0,
18 "puct/buffer_construction_len/mean": 3776.8947368421054,
19 "puct/buffer_construction_len/std": 1930.7091233266558,
20 "puct/buffer_construction_len/min": 1024.0,
21 "puct/buffer_construction_len/max": 32768.0,
22 "puct/sampled_value/mean": 0.9475602048086873,
23 "puct/sampled_value/std": 8.375319120595532e-05,
24 "puct/sampled_value/min": 0.9474764280908119,
25 "puct/sampled_value/max": 0.9476509941872021,
26 "puct/sampled_timestep/mean": 26.0,
27 "puct/sampled_timestep/std": 0.0,
28 "puct/sampled_timestep/min": 26.0,
29 "puct/sampled_timestep/max": 26.0,
30 "puct/sampled_construction_len/mean": 4096.0,
31 "puct/sampled_construction_len/std": 0.0,
32 "puct/sampled_construction_len/min": 4096.0,
33 "puct/sampled_construction_len/max": 4096.0,
34 "time/sampling": 5131.616178035736,
35 "env/all/ac_tokens_per_turn": 8341.916015625,
36 "env/all/ob_tokens_per_turn": 4000.25,
37 "env/all/turns_per_episode": 1.0,
38 "env/all/total_episodes": 512,
39 "env/all/total_turns": 512,
40 "env/all/total_ac_tokens": 4271061,
41 "env/all/total_ob_tokens": 2048128,
42 "env/all/time/sampling_mean": 553.0053381738253,
43 "env/all/time/sampling_max": 759.1140644550323,
44 "env/all/time/env_step_mean": 2411.2443155464716,
45 "env/all/time/env_step_max": 4384.54519033432,
46 "env/all/reward/mean": 0.49716906983777276,
47 "env/all/reward/max": 0.9479329899709945,
48 "env/all/reward/min": 0.0,
49 "env/all/format": 1.0,
50 "env/all/format/min": 1.0,
51 "env/all/format/max": 1.0,
52 "env/all/reward": 0.49716906983777276,
53 "env/all/correctness": 0.556640625,
54 "env/all/correctness/min": 0.0,
55 "env/all/correctness/max": 1.0,
56 "env/all/raw_score": 0.8931598728313671,
57 "env/all/raw_score/min": 0.023147125426254685,
58 "env/all/raw_score/max": 0.9479329899709945,
59 "env/all/initial_raw_score": 0.9475602048086873,
60 "env/all/initial_raw_score/min": 0.9474764280908119,
61 "env/all/initial_raw_score/max": 0.9476509941872021,
62 "env/all/msg": "Success; raw_score=0.9478257617164092",
63 "env/all/parsed_code": "```python\nimport numpy as np\nimport time\nfrom typing import List, Tuple\nimport random\n\ndef _simpson_l2sq(conv: np.ndarray) -> Tuple[float, np.ndarray]:\n m = conv.size\n if m == 0:\n return 0.0, np.zeros_like(conv)\n dx = 1.0 / (m + 1)\n y = np.zeros(m + 2, dtype=conv.dtype)\n y[0] = 0.0\n y[1:-1] = conv\n y[-1] = 0.0\n lhs = y[:-1]\n rhs = y[1:]\n l2_sq = (dx / 3.0) * np.sum(lhs * lhs + lhs * rhs + rhs * rhs)\n grad_y = (dx / 3.0) * (4.0 * y + np.roll(y, 1) + np.roll(y, -1))\n grad_conv = grad_y[1:-1]\n return float(l2_sq), grad_conv\n\ndef _l1(conv: np.ndarray) -> Tuple[float, np.ndarray]:\n m = conv.size\n dx = 1.0 / (m + 1) if m > 0 else 1.0\n val = dx * float(np.sum(conv)) if m > 0 else 0.0\n grad = np.full_like(conv, dx)\n return val, grad\n\ndef _linf(conv: np.ndarray) -> Tuple[float, np.ndarray]:\n if conv.size == 0:\n return 0.0, np.zeros_like(conv)\n m = float(np.max(conv))\n if m <= 0.0:\n return m, np.zeros_like(conv)\n mask = conv == m\n count = int(mask.sum())\n if count == 0:\n return m, np.zeros_like(conv)\n grad = mask.astype(conv.dtype) / count\n return m, grad\n\ndef _objective_and_grad_conv(conv: np.ndarray) -> Tuple[float, np.ndarray]:\n l2_sq, g_l2 = _simpson_l2sq(conv)\n l1, g_l1 = _l1(conv)\n linf, g_linf = _linf(conv)\n if l1 <= 0.0 or linf <= 0.0:\n return 0.0, np.zeros_like(conv)\n denom = l1 * linf\n c_value = l2_sq / denom\n num_grad = g_l2 * denom - l2_sq * (g_l1 * linf + l1 * g_linf)\n g_conv = num_grad / (denom * denom)\n return float(c_value), g_conv\n\ndef _grad_h_from_conv_grad(h: np.ndarray, g_conv: np.ndarray) -> np.ndarray:\n h_rev = h[::-1]\n g_h = np.convolve(g_conv, h_rev, mode=\"valid\")\n return 2.0 * g_h\n\ndef _upsample_1d(h: np.ndarray) -> np.ndarray:\n n = h.shape[0]\n x_old = np.linspace(-0.5, 0.5, n)\n x_new = np.linspace(-0.5, 0.5, 2 * n)\n return np.interp(x_new, x_old, h)\n\ndef _single_candidate_finetune(h0: np.ndarray, lr=0.1, steps=400000, decay_rate=0.999, momentum=0.9, beta=0.999) -> Tuple[np.ndarray, float]:\n h = h0.astype(np.float32).copy()\n last_c = 0.0\n m_buffer = np.zeros_like(h)\n v_buffer = np.zeros_like(h)\n for step in range(steps):\n h_clip = np.clip(h, 0.0, None)\n conv = np.convolve(h_clip, h_clip, mode=\"full\")\n c_val, g_conv = _objective_and_grad_conv(conv)\n g_h = _grad_h_from_conv_grad(h_clip, g_conv)\n m_buffer = momentum * m_buffer + (1 - momentum) * g_h\n v_buffer = beta * v_buffer + (1 - beta) * (g_h ** 2)\n corrected_m = m_buffer / (1 - momentum ** (step + 1))\n corrected_v = v_buffer / (1 - beta ** (step + 1))\n learning_rate_current = lr * (1.0 - step / steps)\n h = np.clip(h + learning_rate_current * corrected_m / (np.sqrt(corrected_v) + 1e-8), 0.0, 1000.0)\n current_c = c_val\n last_c = current_c\n return h, float(last_c)\n\ndef construct_function():\n \"\"\"\n Optimizes a sequence of non-negative heights using a refined multi-scale approach with enhanced exploration, adaptive learning rates,\n and diverse initializations to escape local minima. Starts with multiple diverse initial guesses (structured and random), gradually upscaling\n with adaptive noise and learning rates, and applies fine-tuned adaptive learning rate gradient ascent with momentum.\n \"\"\"\n np.random.seed(42)\n \n # Load previous best sequence if available\n height_sequence_1 = globals().get(\"height_sequence_1\", None)\n target_length = 4096\n initial_n = 8 # Starting sequence length for more diversity\n\n # Generate diverse initial sequences\n if height_sequence_1 is not None:\n # Use previous best and upscale\n initial_h = np.array(height_sequence_1, dtype=np.float32)\n initial_n = min(len(initial_h), target_length)\n x_old = np.linspace(-0.5, 0.5, len(initial_h))\n x_new = np.linspace(-0.5, 0.5, initial_n)\n h = np.interp(x_new, x_old, initial_h)\n else:\n # Generate a few diverse initial sequences\n candidates = []\n # Random candidate\n seq = np.random.rand(target_length) * 0.01 / target_length\n seq_sum = np.sum(seq)\n if seq_sum < 0.01:\n seq = np.clip(seq + (0.01 - seq_sum) / target_length, 0.0, 1000.0)\n candidates.append(seq)\n # Gaussian candidate\n x = np.linspace(-0.5, 0.5, target_length)\n seq_g = 0.01 * np.exp(-((x - 0.0) / 0.2)**2)\n seq_g = np.clip(seq_g / np.sum(seq_g) * 0.01, 0.0, 1000.0)\n candidates.append(seq_g)\n # Multiple peaks candidate\n seq_mp = np.zeros_like(x)\n for i in range(5):\n pos = -0.5 + (i + 0.5) * 0.3\n seq_mp += 0.01 * np.exp(-((x - pos) / 0.1)**2)\n seq_mp = np.clip(seq_mp / np.sum(seq_mp) * 0.01, 0.0, 1000.0)\n candidates.append(seq_mp)\n # Sine wave candidate\n seq_sw = 0.01 * np.sin(2 * np.pi * x * 5)\n seq_sw = np.clip(seq_sw / np.sum(seq_sw) * 0.01, 0.0, 1000.0)\n candidates.append(seq_sw)\n # Triangular candidate\n seq_tr = np.abs(x - 0.0) * (x < 0.5)\n seq_tr = np.clip(seq_tr / np.sum(seq_tr) * 0.01, 0.0, 1000.0)\n candidates.append(seq_tr)\n # Multiple Gaussians candidate\n seq_mp2 = np.zeros_like(x)\n for i in range(7):\n pos = -0.5 + (i + 0.5) * 0.3\n seq_mp2 += 0.01 * np.exp(-((x - pos) / 0.15)**2)\n seq_mp2 = np.clip(seq_mp2 / np.sum(seq_mp2) * 0.01, 0.0, 1000.0)\n candidates.append(seq_mp2)\n # Square wave candidate\n seq_sq = np.zeros_like(x)\n seq_sq[x > 0.0] = 0.01\n seq_sq = np.clip(seq_sq / np.sum(seq_sq) * 0.01, 0.0, 1000.0)\n candidates.append(seq_sq)\n # Single peak at center\n seq_center = np.zeros_like(x)\n seq_center[x < 0.1 and x > -0.1] = 0.01\n seq_center = np.clip(seq_center / np.sum(seq_center) * 0.01, 0.0, 1000.0)\n candidates.append(seq_center)\n # Exponential decay candidate\n seq_exp = 0.01 * np.exp(-x**2)\n seq_exp = np.clip(seq_exp / np.sum(seq_exp) * 0.01, 0.0, 1000.0)\n candidates.append(seq_exp)\n # Wide rectangular pulses\n seq_rect = np.zeros_like(x)\n for i in range(3):\n start = -0.5 + i * 0.5\n end = start + 0.3\n seq_rect[(x >= start) & (x <= end)] += 0.01\n seq_rect = np.clip(seq_rect / np.sum(seq_rect) * 0.01, 0.0, 1000.0)\n candidates.append(seq_rect)\n # Randomized multiple peaks\n seq_rmp = np.zeros_like(x)\n for i in range(10):\n pos = -0.5 + (i + 0.5) * 0.3\n seq_rmp += np.random.uniform(0.001, 0.005) * np.exp(-((x - pos) / 0.1)**2)\n seq_rmp = np.clip(seq_rmp / np.sum(seq_rmp) * 0.01, 0.0, 1000.0)\n candidates.append(seq_rmp)\n # Select the best initial candidate\n best_c = -1.0\n best_h = candidates[0]\n for h_candidate in candidates:\n try:\n c = evaluate_sequence(h_candidate.tolist())\n if c > best_c:\n best_c = c\n best_h = h_candidate\n except:\n pass\n h = best_h\n\n # Parameters for enhanced exploration and refinement\n learning_rate = 0.1\n noise_scale_initial = 100.0 # Increased noise scale for exploration\n noise_decay = 0.995 # Slightly slower noise decay\n learning_rate_decay = 0.9995 # Faster learning rate decay\n max_steps = 600000\n upscale_steps = 130000\n refine_steps = 250000 # Increased steps for better optimization\n\n start_time = time.time()\n\n # Multi-scale optimization with enhanced exploration and multiple initializations\n current_length = initial_n\n scale_factor = 2\n\n for _ in range(6): # Upscale 6 times to reach 4096\n if current_length < target_length:\n h = _upsample_1d(h)\n current_length *= 2\n # Ensure sum is above 0.01\n h_sum = np.sum(h)\n if h_sum < 0.01:\n h = np.clip(h + (0.01 - h_sum) / h.shape[0], 0.0, 1000.0)\n\n # Initial optimization with dynamic noise and learning rate\n h_opt = np.copy(h)\n h_best = np.copy(h)\n best_c = -1.0\n for step in range(upscale_steps): # Increased steps for better optimization\n # Clip to non-negative\n clipped_h = np.clip(h_opt, 0.0, None)\n\n # Compute convolution\n conv = np.convolve(clipped_h, clipped_h, mode='full')\n\n # Compute objective and gradient of conv\n obj_val, grad_conv = _objective_and_grad_conv(conv)\n\n # Compute gradient with respect to h\n grad_h = _grad_h_from_conv_grad(clipped_h, grad_conv)\n\n # Adjust noise and learning rate based on step\n noise_scale = noise_scale_initial * (noise_decay ** step)\n learning_rate_current = learning_rate * (learning_rate_decay ** step)\n\n # Add noise\n noise = noise_scale * np.random.normal(size=h_opt.shape)\n\n # Update h with adaptive noise and learning\n h_opt = np.clip(h_opt + learning_rate_current * grad_h + noise, 0.0, 1000.0)\n\n # Maintain sum requirement dynamically\n h_sum_current = np.sum(h_opt)\n if h_sum_current < 0.01:\n scale_factor = 0.01 / h_sum_current\n h_opt = np.clip(h_opt * scale_factor, 0.0, 1000.0)\n\n # Periodic evaluation and update best\n if step % 500 == 0:\n try:\n current_c = evaluate_sequence(h_opt.tolist())\n if current_c > best_c:\n best_c = current_c\n h_best = h_opt.copy()\n except:\n pass\n\n # Check time remaining\n remaining_time = 1000 - (time.time() - start_time)\n if remaining_time < 5:\n print(f\"Time remaining: {remaining_time} seconds. Performing final refinement.\")\n break\n # Assign optimized h\n h = h_best\n\n # Final refined optimization with smaller learning rate and adaptive decay\n h_refined, _ = _single_candidate_finetune(h, lr=0.1, steps=refine_steps, decay_rate=0.999, momentum=0.9, beta=0.999)\n\n # Final check and normalization\n h_final = np.clip(h_refined, 0.0, 1000.0)\n heights = h_final.tolist()\n r_value = evaluate_sequence(heights)\n print(f\"Final C2 lower bound: {r_value}\")\n return heights\n```",
64 "env/all/time/policy": 553.0053381738253,
65 "env/all/time/policy/min": 278.10565304756165,
66 "env/all/time/policy/max": 759.1140644550323,
67 "env/all/time/env_step": 2411.2443155464716,
68 "env/all/time/env_step/min": 0.006899118423461914,
69 "env/all/time/env_step/max": 4384.54519033432,
70 "env/all/time/reward_compute": 4.0652230381965637e-07,
71 "env/all/time/reward_compute/min": 1.5273690223693848e-07,
72 "env/all/time/reward_compute/max": 8.083879947662354e-07,
73 "env/all/by_group/frac_mixed": 1.0,
74 "env/all/by_group/frac_all_good": 0.0,
75 "env/all/by_group/frac_all_bad": 0.0,
76 "advantage/mean": 0.02429594099521637,
77 "advantage/min": -1.0,
78 "advantage/max": 9.403131484985352,
79 "time/assemble_training_data": 9.80262017250061,
80 "time/kl_vs_base": 150.32044553756714,
81 "kl_policy_base": 0.0007301223813556135,
82 "time/train": 1187.3255634307861,
83 "time/save_checkpoint": 12.120770454406738,
84 "time/total": 6495.393157243729
85}[2026-07-09T06:42:12+00:00] job=1812634 node=node-30 ngpu=3 ntrain=1 replicas=2 flash_attn=no
[2026-07-09T07:26:33+00:00] job=1812736 node=node-12 ngpu=3 ntrain=1 replicas=2 flash_attn=yes
[2026-07-09T09:27:32+00:00] job=1813133 node=node-14 ngpu=3 ntrain=1 replicas=2 flash_attn=yes
[2026-07-09T16:26:04+00:00] job=1813134 node=node-4 ngpu=6 ntrain=2 replicas=4 flash_attn=yes
[2026-07-11T01:56:04+00:00] job=1821290 node=node-4 ngpu=3 ntrain=1 replicas=2 flash_attn=yes