Views
No views yet
ac2. Checkpoint saved
after training step 26 (0-indexed). Strict upstream eval parity:
1100s hard kill, verbatim prompts/entrypoints, group 64x8, T=1.0, kl 0.1.1{
2 "step": 26,
3 "progress/batch": 26,
4 "optim/lr": 4e-05,
5 "progress/done_frac": 0.54,
6 "puct/buffer_size": 421,
7 "puct/sampled_size": 8,
8 "puct/T": 13312,
9 "puct/scale_last": 0.28090836937721664,
10 "puct/buffer_value/mean": 0.9318236464903161,
11 "puct/buffer_value/std": 0.044175651820447644,
12 "puct/buffer_value/min": 0.6666666666666636,
13 "puct/buffer_value/max": 0.9475750360438833,
14 "puct/buffer_timestep/mean": 12.311163895486937,
15 "puct/buffer_timestep/std": 7.638855159415902,
16 "puct/buffer_timestep/min": -1.0,
17 "puct/buffer_timestep/max": 25.0,
18 "puct/buffer_construction_len/mean": 3764.7672209026127,
19 "puct/buffer_construction_len/std": 1966.0337306107954,
20 "puct/buffer_construction_len/min": 1024.0,
21 "puct/buffer_construction_len/max": 32768.0,
22 "puct/sampled_value/mean": 0.9474993145368209,
23 "puct/sampled_value/std": 4.121155703567937e-05,
24 "puct/sampled_value/min": 0.9474754652434118,
25 "puct/sampled_value/max": 0.9475750360438833,
26 "puct/sampled_timestep/mean": 25.0,
27 "puct/sampled_timestep/std": 0.0,
28 "puct/sampled_timestep/min": 25.0,
29 "puct/sampled_timestep/max": 25.0,
30 "puct/sampled_construction_len/mean": 4096.0,
31 "puct/sampled_construction_len/std": 0.0,
32 "puct/sampled_construction_len/min": 4096.0,
33 "puct/sampled_construction_len/max": 4096.0,
34 "time/sampling": 4703.886113643646,
35 "env/all/ac_tokens_per_turn": 8447.591796875,
36 "env/all/ob_tokens_per_turn": 3953.0,
37 "env/all/turns_per_episode": 1.0,
38 "env/all/total_episodes": 512,
39 "env/all/total_turns": 512,
40 "env/all/total_ac_tokens": 4325167,
41 "env/all/total_ob_tokens": 2023936,
42 "env/all/time/sampling_mean": 565.5829308913089,
43 "env/all/time/sampling_max": 781.3132710456848,
44 "env/all/time/env_step_mean": 1837.2060738503933,
45 "env/all/time/env_step_max": 3929.165500164032,
46 "env/all/reward/mean": 0.5483584532131192,
47 "env/all/reward/max": 0.9476509941872021,
48 "env/all/reward/min": 0.0,
49 "env/all/format": 1.0,
50 "env/all/format/min": 1.0,
51 "env/all/format/max": 1.0,
52 "env/all/reward": 0.5483584532131192,
53 "env/all/correctness": 0.630859375,
54 "env/all/correctness/min": 0.0,
55 "env/all/correctness/max": 1.0,
56 "env/all/raw_score": 0.8692245450313222,
57 "env/all/raw_score/min": 0.014351168059386217,
58 "env/all/raw_score/max": 0.9476509941872021,
59 "env/all/initial_raw_score": 0.9474993145368211,
60 "env/all/initial_raw_score/min": 0.9474754652434118,
61 "env/all/initial_raw_score/max": 0.9475750360438833,
62 "env/all/msg": "Evaluation timed out after 1100 seconds.",
63 "env/all/parsed_code": "```python\nimport numpy as np\nimport time\nfrom typing import List, Tuple\nimport random\n\ndef _simpson_l2sq(conv: np.ndarray) -> Tuple[float, np.ndarray]:\n m = conv.size\n if m == 0:\n return 0.0, np.zeros_like(conv)\n dx = 1.0 / (m + 1)\n y = np.zeros(m + 2, dtype=conv.dtype)\n y[0] = 0.0\n y[1:-1] = conv\n y[-1] = 0.0\n lhs = y[:-1]\n rhs = y[1:]\n l2_sq = (dx / 3.0) * np.sum(lhs * lhs + lhs * rhs + rhs * rhs)\n grad_y = (dx / 3.0) * (4.0 * y + np.roll(y, 1) + np.roll(y, -1))\n grad_conv = grad_y[1:-1]\n return float(l2_sq), grad_conv\n\ndef _l1(conv: np.ndarray) -> Tuple[float, np.ndarray]:\n m = conv.size\n dx = 1.0 / (m + 1) if m > 0 else 1.0\n val = dx * float(np.sum(conv)) if m > 0 else 0.0\n grad = np.full_like(conv, dx)\n return val, grad\n\ndef _linf(conv: np.ndarray) -> Tuple[float, np.ndarray]:\n if conv.size == 0:\n return 0.0, np.zeros_like(conv)\n m = float(np.max(conv))\n if m <= 0.0:\n return m, np.zeros_like(conv)\n mask = conv == m\n count = int(mask.sum())\n if count == 0:\n return m, np.zeros_like(conv)\n grad = mask.astype(conv.dtype) / count\n return m, grad\n\ndef _objective_and_grad_conv(conv: np.ndarray) -> Tuple[float, np.ndarray]:\n l2_sq, g_l2 = _simpson_l2sq(conv)\n l1, g_l1 = _l1(conv)\n linf, g_linf = _linf(conv)\n if l1 <= 0.0 or linf <= 0.0:\n return 0.0, np.zeros_like(conv)\n denom = l1 * linf\n c_value = l2_sq / denom\n num_grad = g_l2 * denom - l2_sq * (g_l1 * linf + l1 * g_linf)\n g_conv = num_grad / (denom * denom)\n return float(c_value), g_conv\n\ndef _grad_h_from_conv_grad(h: np.ndarray, g_conv: np.ndarray) -> np.ndarray:\n h_rev = h[::-1]\n g_h = np.convolve(g_conv, h_rev, mode=\"valid\")\n return 2.0 * g_h\n\ndef _upsample_1d(h: np.ndarray) -> np.ndarray:\n n = h.shape[0]\n x_old = np.linspace(-0.5, 0.5, n)\n x_new = np.linspace(-0.5, 0.5, 2 * n)\n return np.interp(x_new, x_old, h)\n\ndef _single_candidate_finetune(h0: np.ndarray, lr=0.01, steps=300000, decay_rate=0.9995, momentum=0.9, beta=0.999) -> Tuple[np.ndarray, float]:\n h = h0.astype(np.float32).copy()\n last_c = 0.0\n m_buffer = np.zeros_like(h)\n v_buffer = np.zeros_like(h)\n for step in range(steps):\n h_clip = np.clip(h, 0.0, None)\n conv = np.convolve(h_clip, h_clip, mode=\"full\")\n c_val, g_conv = _objective_and_grad_conv(conv)\n g_h = _grad_h_from_conv_grad(h_clip, g_conv)\n # Momentum update\n m_buffer = momentum * m_buffer + (1 - momentum) * g_h\n v_buffer = beta * v_buffer + (1 - beta) * (g_h ** 2)\n corrected_m = m_buffer / (1 - momentum ** (step + 1))\n corrected_v = v_buffer / (1 - beta ** (step + 1))\n learning_rate_current = lr * (1.0 - step / steps)\n h = np.clip(h + learning_rate_current * corrected_m / (np.sqrt(corrected_v) + 1e-8), 0.0, 1000.0)\n current_c = c_val\n last_c = current_c\n return h, float(last_c)\n\ndef construct_function():\n \"\"\"\n Optimizes a sequence of non-negative heights using a refined multi-scale approach with enhanced exploration, adaptive learning rates,\n and diverse initializations to escape local minima. Starts with multiple diverse initial guesses (structured and random), gradually upscaling\n with adaptive noise and learning rates, and applies fine-tuned adaptive learning rate gradient ascent with momentum.\n \"\"\"\n np.random.seed(42)\n \n # Load previous best sequence if available\n height_sequence_1 = globals().get(\"height_sequence_1\", None)\n target_length = 4096\n initial_n = 8 # Starting sequence length for more diversity\n\n # Generate diverse initial sequences\n if height_sequence_1 is not None:\n # Use previous best and upscale\n initial_h = np.array(height_sequence_1, dtype=np.float32)\n initial_n = min(len(initial_h), target_length)\n x_old = np.linspace(-0.5, 0.5, len(initial_h))\n x_new = np.linspace(-0.5, 0.5, initial_n)\n h = np.interp(x_new, x_old, initial_h)\n else:\n # Generate a few diverse initial sequences\n candidates = []\n # Random candidate\n seq = np.random.rand(target_length) * 0.01 / target_length\n seq_sum = np.sum(seq)\n if seq_sum < 0.01:\n seq = np.clip(seq + (0.01 - seq_sum) / target_length, 0.0, 1000.0)\n candidates.append(seq)\n # Gaussian candidate\n x = np.linspace(-0.5, 0.5, target_length)\n seq_g = 0.01 * np.exp(-((x - 0.0) / 0.2)**2)\n seq_g = np.clip(seq_g / np.sum(seq_g) * 0.01, 0.0, 1000.0)\n candidates.append(seq_g)\n # Multiple peaks candidate\n seq_mp = np.zeros_like(x)\n for i in range(3):\n pos = -0.5 + (i + 0.5) * 0.3\n seq_mp += 0.01 * np.exp(-((x - pos) / 0.1)**2)\n seq_mp = np.clip(seq_mp / np.sum(seq_mp) * 0.01, 0.0, 1000.0)\n candidates.append(seq_mp)\n # Sine wave candidate\n seq_sw = 0.01 * np.sin(2 * np.pi * x * 5)\n seq_sw = np.clip(seq_sw / np.sum(seq_sw) * 0.01, 0.0, 1000.0)\n candidates.append(seq_sw)\n # Triangular candidate\n seq_tr = np.abs(x - 0.0) * (x < 0.5)\n seq_tr = np.clip(seq_tr / np.sum(seq_tr) * 0.01, 0.0, 1000.0)\n candidates.append(seq_tr)\n # Single spike candidate\n seq_spike = np.zeros_like(x)\n spike_height = 0.01\n seq_spike[target_length // 2] = spike_height\n seq_spike = np.clip(seq_spike, 0.0, 1000.0)\n candidates.append(seq_spike)\n # Decaying exponential candidate\n seq_decay = 0.01 * np.exp(-np.abs(x) * 5)\n seq_decay = np.clip(seq_decay / np.sum(seq_decay) * 0.01, 0.0, 1000.0)\n candidates.append(seq_decay)\n # Multiple spaced peaks candidate\n seq_multi = np.zeros_like(x)\n for i in range(5):\n pos = -0.5 + (i + 0.5) * 0.3\n seq_multi += 0.01 * np.exp(-((x - pos) / 0.25)**2)\n seq_multi = np.clip(seq_multi / np.sum(seq_multi) * 0.01, 0.0, 1000.0)\n candidates.append(seq_multi)\n # Flat line candidate\n seq_flat = np.ones_like(x) * (0.01 / target_length)\n seq_flat = np.clip(seq_flat, 0.0, 1000.0)\n candidates.append(seq_flat)\n # Select the best initial candidate\n best_c = -1.0\n best_h = candidates[0]\n for h_candidate in candidates:\n try:\n c = evaluate_sequence(h_candidate.tolist())\n if c > best_c:\n best_c = c\n best_h = h_candidate\n except:\n pass\n h = best_h\n\n # Parameters for enhanced exploration and refinement\n learning_rate = 0.1\n noise_scale_initial = 15.0 # Increased noise scale for exploration\n noise_decay = 0.997 # Slower noise decay\n learning_rate_decay = 0.9998 # Faster learning rate decay\n max_steps = 500000\n upscale_steps = 200000 # Increased\n refine_steps = 400000 # Increased\n\n start_time = time.time()\n\n # Multi-scale optimization with enhanced exploration and multiple initializations\n current_length = initial_n\n scale_factor = 2\n\n for _ in range(6): # Upscale 6 times to reach 4096\n if current_length < target_length:\n h = _upsample_1d(h)\n current_length *= 2\n # Ensure sum is above 0.01\n h_sum = np.sum(h)\n if h_sum < 0.01:\n h = np.clip(h + (0.01 - h_sum) / h.shape[0], 0.0, 1000.0)\n\n # Initial optimization with dynamic noise and learning rate\n h_opt = np.copy(h)\n h_best = np.copy(h)\n best_c = -1.0\n for step in range(upscale_steps): # Increased steps for better optimization\n # Clip to non-negative\n clipped_h = np.clip(h_opt, 0.0, None)\n\n # Compute convolution\n conv = np.convolve(clipped_h, clipped_h, mode='full')\n\n # Compute objective and gradient of conv\n obj_val, grad_conv = _objective_and_grad_conv(conv)\n\n # Compute gradient with respect to h\n grad_h = _grad_h_from_conv_grad(clipped_h, grad_conv)\n\n # Adjust noise and learning rate based on step\n noise_scale = noise_scale_initial * (noise_decay ** step)\n learning_rate_current = learning_rate * (learning_rate_decay ** step)\n\n # Add noise\n noise = noise_scale * np.random.normal(size=h_opt.shape)\n\n # Update h with adaptive noise and learning\n h_opt = np.clip(h_opt + learning_rate_current * grad_h + noise, 0.0, 1000.0)\n\n # Maintain sum requirement dynamically\n h_sum_current = np.sum(h_opt)\n if h_sum_current < 0.01:\n scale_factor = 0.01 / h_sum_current\n h_opt = np.clip(h_opt * scale_factor, 0.0, 1000.0)\n\n # Periodic evaluation and update best\n if step % 200 == 0: # Evaluate more frequently\n try:\n current_c = evaluate_sequence(h_opt.tolist())\n if current_c > best_c:\n best_c = current_c\n h_best = h_opt.copy()\n except:\n pass\n # Check time remaining\n remaining_time = 1000 - (time.time() - start_time)\n if remaining_time < 5:\n print(f\"Time remaining: {remaining_time} seconds. Performing final refinement.\")\n break\n # Assign optimized h\n h = h_best\n else:\n break\n\n # Final refined optimization with smaller learning rate and adaptive decay\n h_refined, _ = _single_candidate_finetune(h, lr=0.01, steps=refine_steps, decay_rate=0.9995)\n\n # Final check and normalization\n h_final = np.clip(h_refined, 0.0, 1000.0)\n heights = h_final.tolist()\n r_value = evaluate_sequence(heights)\n print(f\"Final C2 lower bound: {r_value}\")\n return heights\n```",
64 "env/all/time/policy": 565.5829308913089,
65 "env/all/time/policy/min": 269.77454590797424,
66 "env/all/time/policy/max": 781.3132710456848,
67 "env/all/time/env_step": 1837.2060738503933,
68 "env/all/time/env_step/min": 0.00675511360168457,
69 "env/all/time/env_step/max": 3929.165500164032,
70 "env/all/time/reward_compute": 3.380700945854187e-07,
71 "env/all/time/reward_compute/min": 1.7136335372924805e-07,
72 "env/all/time/reward_compute/max": 1.1883676052093506e-06,
73 "env/all/by_group/frac_mixed": 1.0,
74 "env/all/by_group/frac_all_good": 0.0,
75 "env/all/by_group/frac_all_bad": 0.0,
76 "advantage/mean": 0.018338236957788467,
77 "advantage/min": -1.0,
78 "advantage/max": 6.837209224700928,
79 "time/assemble_training_data": 8.960397005081177,
80 "time/kl_vs_base": 149.4393789768219,
81 "kl_policy_base": 0.0006924000335857272,
82 "time/train": 1195.6682057380676,
83 "time/save_checkpoint": 18.069377422332764,
84 "time/total": 6080.260942697525
85}[2026-07-09T06:42:12+00:00] job=1812634 node=node-30 ngpu=3 ntrain=1 replicas=2 flash_attn=no
[2026-07-09T07:26:33+00:00] job=1812736 node=node-12 ngpu=3 ntrain=1 replicas=2 flash_attn=yes
[2026-07-09T09:27:32+00:00] job=1813133 node=node-14 ngpu=3 ntrain=1 replicas=2 flash_attn=yes
[2026-07-09T16:26:04+00:00] job=1813134 node=node-4 ngpu=6 ntrain=2 replicas=4 flash_attn=yes
[2026-07-11T01:56:04+00:00] job=1821290 node=node-4 ngpu=3 ntrain=1 replicas=2 flash_attn=yes