Views
No views yet
1# ARC training config
2
3defaults:
4 - arch: trm
5 - _self_
6
7hydra:
8 output_subdir: null
9
10entity: "trelis"
11
12# Data path
13data_paths: ['data/arc2concept-aug-1000']
14data_paths_test: []
15
16evaluators:
17 - name: arc@ARC
18
19# Hyperparams - Training
20global_batch_size: 768
21
22epochs: 100000
23eval_interval: 10000
24checkpoint_every_eval: True
25checkpoint_every_n_steps: null
26
27lr: 1e-4
28lr_min_ratio: 1.0
29lr_warmup_steps: 2000
30
31# Standard hyperparameter settings for LM, as used in Llama
32beta1: 0.9
33beta2: 0.95
34weight_decay: 0.1
35puzzle_emb_weight_decay: 0.1
36
37# Hyperparams - Puzzle embeddings training
38puzzle_emb_lr: 1e-2
39
40seed: 0
41min_eval_interval: 0 # when to start the eval
42
43ema: True # use Exponential-Moving-Average
44ema_rate: 0.999 # EMA-rate
45freeze_weights: False # If True, freeze weights and only learn the embeddings1name: recursive_reasoning.trm@TinyRecursiveReasoningModel_ACTV1
2loss:
3 name: losses@ACTLossHead
4 loss_type: stablemax_cross_entropy
5
6halt_exploration_prob: 0.1
7halt_max_steps: 16
8
9H_cycles: 3
10L_cycles: 4 # NOTE THAT THIS IS DIFFERENT THAN THE PAPER, THAT USES 6. THE DIFFERENCE WAS ACCIDENTAL.
11
12H_layers: 0
13L_layers: 2
14
15hidden_size: 512
16num_heads: 8 # min(2, hidden_size // 64)
17expansion: 4
18
19puzzle_emb_ndim: ${.hidden_size}
20
21pos_encodings: rope
22forward_dtype: bfloat16
23
24mlp_t: False # use mlp on L instead of transformer
25puzzle_emb_len: 16 # if non-zero, its specified to this value
26no_ACT_continue: True # No continue ACT loss, only use the sigmoid of the halt which makes much more sense