test/params_XLM-R.json at main · WoutPiessens/test · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
{
  "transformer_model": "xlm-roberta-large",
  "random_seed": 8446,
  "default_dec_dataset_embeds_dim": 12,
  "encoder": {
    "dropout": 0.2,
    "max_input_length": 128,
    "update_weights_encoder": true
  },
  "decoders": {
    "default_decoder": {
      "loss_weight": 1.0,
      "metric": "accuracy",
      "topn": 1,
      "layers_to_use": [-1]
    },
    "classification": {
    },
    "dependency": {
      "arc_representation_dim": 768,
      "tag_representation_dim": 256,
      "metric": "las"
    },
    "mlm": {
      "metric": "perplexity"
    },
    "multiclas": {
      "metric": "multi_acc",
      "threshold": 0.7
    },
    "multiseq": {
      "metric": "multi_acc",
      "threshold": 0.7
    },
    "regression": {
      "metric": "avg_dist"
    },
    "seq": {
    },
    "seq_bio": {
      "metric": "span_f1"
    },
    "string2string": {
    },
    "tok": {
      "pre_split": true
    }
  },
  "batching": {
    "max_tokens": 1024,
    "batch_size": 32,
    "sort_by_size": true,
    "sampling_smoothing": 1.0 // 1.0 == original size, 0.0==all equal
  },
  "training": {
    "keep_top_n": 1,
    "learning_rate_scheduler": {
      //"type": "slanted_triangular",
      "cut_frac": 0.3,
      "decay_factor": 0.38,
      "discriminative_fine_tuning": true,
      "gradual_unfreezing": true
    },
    "num_epochs": 20,
    "optimizer": {
      //"type": "adamw",
      "betas": [
        0.9,
        0.99
      ],
      "lr": 0.0001,
      "correct_bias": false,
      //"patience": 5, // disabled, because slanted_triangular changes the lr dynamically
      "weight_decay": 0.01
    }
  }
}