| { | |
| "model": { | |
| "num_labels": 1641, | |
| "max_len": 256, | |
| "d_model": 1024, | |
| "n_layers": 12, | |
| "n_heads": 16, | |
| "dropout": 0.1, | |
| "conv_kernel": 5, | |
| "downsample": 2, | |
| "ngram_buckets": 32768, | |
| "script_buckets": 512, | |
| "rope_theta": 10000.0, | |
| "vocab_size": 259, | |
| "modalities": [ | |
| "text" | |
| ], | |
| "model_type": "millid-byte-encoder" | |
| }, | |
| "training": { | |
| "dataset": "baobabtech/millid-dataset", | |
| "preset": "large", | |
| "max_len": 256, | |
| "steps": 120000, | |
| "batch_size": 512, | |
| "lr": 0.00015, | |
| "warmup": 2000, | |
| "weight_decay": 0.01, | |
| "dropout": 0.1, | |
| "label_smoothing": 0.1, | |
| "alpha": 0.3, | |
| "short_p": 0.25, | |
| "eval_every": 2000, | |
| "eval_per_label": 20, | |
| "patience": 30000, | |
| "commercial_only": false, | |
| "extra": [], | |
| "compare_glotlid": true, | |
| "compare_fasttext": true, | |
| "final_eval": "all", | |
| "output_dir": "millid-out", | |
| "push_to": "baobabtech/millid-large-v1.5", | |
| "public": false, | |
| "seed": 7, | |
| "log_every": 100, | |
| "threads": 0, | |
| "workers": -1, | |
| "compile": true, | |
| "gpus": 0, | |
| "wandb_project": "millid", | |
| "wandb_entity": "mscsr000324-must", | |
| "wandb_run_name": "millid-large-v1.5", | |
| "wandb_tags": [ | |
| "large", | |
| "v1.5" | |
| ], | |
| "no_wandb": false | |
| }, | |
| "best_step": 114000, | |
| "parameters": 192636521, | |
| "gpus": 4, | |
| "train_passages": 1943957, | |
| "trained_labels": 1641, | |
| "seconds": 8128 | |
| } |