Views
No views yet
1{
2 "data_dir": "/raid/konstantin.dobler/culturax/ar/ar/tokenized/tokenizers_ar_sp-bpe-ar-32kauto",
3 "model_path": "/raid/konstantin.dobler/checkpoints/culturax-ar-spbpe32k-smart-heuristics-attn-fix-infini-just-embs/ruxr78xn/step-0000100-ckpt",
4 "from_scratch": false,
5 "saved_checkpoint_path": null,
6 "resume": false,
7 "train_file": "train.txt",
8 "val_file": "val.txt",
9 "tokenizer_path": "/raid/konstantin.dobler/checkpoints/culturax-ar-spbpe32k-smart-heuristics-attn-fix-infini-just-embs/ruxr78xn/step-0000100-ckpt",
10 "base_unit": "optimizer-steps",
11 "training_goal": 7680,
12 "eval_interval": 384,
13 "eval_samples": 10000,
14 "save_interval": 768,
15 "log_interval": 1,
16 "model_profiling_interval": 10,
17 "warmup_period": 76,
18 "lr_decay_period": 4608,
19 "lr_final_annealing_period": 1075,
20 "block_size": 4096,
21 "decontaminated_packing": true,
22 "max_lr": 3e-05,
23 "batch_size": 256,
24 "weight_decay": 0.05,
25 "beta1": 0.9,
26 "beta2": 0.95,
27 "grad_clip": 1.0,
28 "min_lr": 2e-06,
29 "infinite_lr": 1.5e-05,
30 "accelerator": "cuda",
31 "num_devices": 4,
32 "activation_checkpointing": false,
33 "distributed_strategy": "auto",
34 "use_fsdp": true,
35 "fsdp_sharding_strategy": "SHARD_GRAD_OP",
36 "fsdp_limit_all_gathers": false,
37 "fsdp_cpu_offload": false,
38 "fsdp_ram_friendly_checkpointing": false,
39 "fsdp_backward_prefetch_post": false,
40 "smart_cuda_alloc": false,
41 "fast_model_loading": true,
42 "micro_batch_size": 1,
43 "eval_micro_batch_size": 10,
44 "gradient_accumulation_steps": 64,
45 "precision": "bf16-true",
46 "use_anyprecision_adamw": false,
47 "adamw_foreach": true,
48 "compile": false,
49 "use_additional_flash_attn_kernels": true,
50 "workers": 8,
51 "preprocessing_workers": 224,
52 "run_name": "culturax-ar-spbpe32k-focus+justembs-attn-fix-infini",
53 "seed": 42,
54 "only_val": false,
55 "val_before_training": true,
56 "out_dir": "/raid/konstantin.dobler/checkpoints/culturax-ar-spbpe32k-focus+justembs-attn-fix-infini",
57 "wandb_tags": [],
58 "offline": false,
59 "debug": false,
60 "model_profiling": true,
61 "force_deterministic": false,
62 "fast_dev_run": false,
63 "cross_tokenizer_val": false,
64 "optimized_activation_checkpointing_policy": false,
65 "train_embeddings": false,
66 "train_only_embeddings": false,
67 "focus_init": false,
68 "refocus_init": false,
69 "mean_init": false,
70 "random_init": false,
71 "zipf_init": false,
72 "smart_heuristic_init": false,
73 "wechsel_init": false,
74 "deepfocus_init": false,
75 "zett_init": false,
76 "focus_fasttext_dim": 300,
77 "focus_fasttext_epochs": 3,
78 "focus_fasttext_min_count": 10,
79 "focus_auxiliary_mode": "fasttext-tokenlevel",
80 "focus_fasttext_model_path": null,
81 "focus_exact_match_all": false,
82 "focus_match_symbols": false,
83 "focus_bilingual_dict": null,
84 "focus_bilingual_dict_mode": "mean",
85 "focus_fuzzy_match_all": false,
86 "focus_random_init_source": "source",
87 "lora_r": 8,
88 "lora_alpha": 16,
89 "lora_dropout": 0.05,
90 "lora_query": false,
91 "lora_key": false,
92 "lora_value": false,
93 "lora_projection": false,
94 "lora_mlp": false,
95 "lora_head": false,
96 "perf_benchmark": false
97}