LM-Trainer/configs/cpu_small_model.json at main · Modotte/LM-Trainer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
{
  "model": {
    "vocab_size": 32000,
    "d_model": 256,
    "n_heads": 4,
    "n_layers": 4,
    "d_ff": 1024,
    "max_seq_len": 512,
    "dropout": 0.1,
    "attention_dropout": 0.1,
    "init_std": 0.02,
    "use_learned_pos_emb": false,
    "layer_norm_eps": 1e-5,
    "pre_norm": true,
    "activation": "gelu",
    "use_bias": true,
    "gradient_checkpointing": false
  },
  "training": {
    "batch_size": 2,
    "learning_rate": 8e-4,
    "weight_decay": 0.01,
    "num_epochs": 3,
    "lr_scheduler": "cosine",
    "warmup_steps": 250,
    "warmup_ratio": 0.1,
    "optimizer": "adamw",
    "max_grad_norm": 1.0,
    "gradient_accumulation_steps": 8,
    "use_amp": false,
    "save_steps": 500,
    "save_total_limit": 3,
    "checkpoint_dir": "./checkpoints/cpu_small_model",
    "eval_steps": 250,
    "eval_strategy": "steps",
    "logging_steps": 25,
    "report_to": ["tensorboard"],
    "dataloader_num_workers": 2,
    "dataloader_pin_memory": false,
    "dataloader_drop_last": true,
    "device": "cpu",
    "force_cpu": true,
    "compile_model": false,
    "auto_tune_batch_size": false,
    "seed": 42
  }
}