| { |
| "resume_count": 0, |
| "stages": { |
| "random_c4": { |
| "resume_count": 0, |
| "initialization_notes": [ |
| "Initializing model from base model: /cephfs/user/nklugeco/checkpoints/random/reset_model for continual pretraining/fine-tuning.", |
| "Applied Liger kernels to the model.", |
| "Number of trainable parameters: 670,127,616", |
| "Shuffling enabled. Shuffling 36 dataset files.", |
| "Collate function will mask token IDs: [0]", |
| "Overriding the number of steps to 10000 as per the `max_steps` argument (check the YAML file if you are not sure).", |
| "Using learning rate decay type: cosine" |
| ], |
| "events": [ |
| "Learning rate stage changed to: cosine_decay at step 1000 | random_c4." |
| ], |
| "run_info": { |
| "Training stage": "random_c4", |
| "Run ID": "107717", |
| "Hardware": "H200", |
| "World size (total GPUs)": "2", |
| "Precision": "bfloat16" |
| }, |
| "Dataset Configuration": { |
| "Num train examples": "2,612,686", |
| "Num validation examples": "8,192", |
| "Length of train dataloader": "81,647", |
| "Max position embeddings (seq length)": "4,096", |
| "Shuffle dataset": "True", |
| "Masked token IDs": "[0]" |
| }, |
| "Batch Configuration": { |
| "Num Epochs": "1", |
| "Micro batch size per device": "16", |
| "Gradient accumulation steps": "4", |
| "Total batch size (samples)": "128", |
| "Total batch size (tokens)": "524,288", |
| "Total optimization steps": "10,000", |
| "Steps per epoch": "20,412", |
| "Checkpointing every": "5000 steps" |
| }, |
| "Model Architecture": { |
| "Model config": "/cephfs/user/nklugeco/checkpoints/random/reset_model", |
| "Attention implementation": "flash_attention_4", |
| "Gradient checkpointing": "False", |
| "Liger kernel": "True", |
| "Torch compile": "False", |
| "Trainable parameters": "670,127,616" |
| }, |
| "Optimizer Configuration (AdamW)": { |
| "Optimizer type": "adamw", |
| "Max learning rate (Adam)": "0.0005", |
| "Min learning rate": "0.0", |
| "LR scheduler type": "COSINE", |
| "LR decay iterations coef": "0.1", |
| "Warmup steps": "1000", |
| "Weight decay": "0.1", |
| "Beta1": "0.9", |
| "Beta2": "0.95", |
| "Epsilon": "1e-08", |
| "Max grad norm": "1.0" |
| } |
| } |
| }, |
| "emissions": { |
| "duration_hours": 8.1735, |
| "energy_consumed_kwh": 3.975328, |
| "co2_emissions_kgco2eq": 1.514401, |
| "num_nodes": 1 |
| } |
| } |
