{
  "resume_count": 0,
  "stages": {
    "random_c4": {
      "resume_count": 0,
      "initialization_notes": [
        "Initializing model from base model: /cephfs/user/nklugeco/checkpoints/random/reset_model for continual pretraining/fine-tuning.",
        "Applied Liger kernels to the model.",
        "Number of trainable parameters: 670,127,616",
        "Shuffling enabled. Shuffling 36 dataset files.",
        "Collate function will mask token IDs: [0]",
        "Overriding the number of steps to 10000 as per the `max_steps` argument (check the YAML file if you are not sure).",
        "Using learning rate decay type: cosine"
      ],
      "events": [
        "Learning rate stage changed to: cosine_decay at step 1000 | random_c4."
      ],
      "run_info": {
        "Training stage": "random_c4",
        "Run ID": "107717",
        "Hardware": "H200",
        "World size (total GPUs)": "2",
        "Precision": "bfloat16"
      },
      "Dataset Configuration": {
        "Num train examples": "2,612,686",
        "Num validation examples": "8,192",
        "Length of train dataloader": "81,647",
        "Max position embeddings (seq length)": "4,096",
        "Shuffle dataset": "True",
        "Masked token IDs": "[0]"
      },
      "Batch Configuration": {
        "Num Epochs": "1",
        "Micro batch size per device": "16",
        "Gradient accumulation steps": "4",
        "Total batch size (samples)": "128",
        "Total batch size (tokens)": "524,288",
        "Total optimization steps": "10,000",
        "Steps per epoch": "20,412",
        "Checkpointing every": "5000 steps"
      },
      "Model Architecture": {
        "Model config": "/cephfs/user/nklugeco/checkpoints/random/reset_model",
        "Attention implementation": "flash_attention_4",
        "Gradient checkpointing": "False",
        "Liger kernel": "True",
        "Torch compile": "False",
        "Trainable parameters": "670,127,616"
      },
      "Optimizer Configuration (AdamW)": {
        "Optimizer type": "adamw",
        "Max learning rate (Adam)": "0.0005",
        "Min learning rate": "0.0",
        "LR scheduler type": "COSINE",
        "LR decay iterations coef": "0.1",
        "Warmup steps": "1000",
        "Weight decay": "0.1",
        "Beta1": "0.9",
        "Beta2": "0.95",
        "Epsilon": "1e-08",
        "Max grad norm": "1.0"
      }
    }
  },
  "emissions": {
    "duration_hours": 8.1735,
    "energy_consumed_kwh": 3.975328,
    "co2_emissions_kgco2eq": 1.514401,
    "num_nodes": 1
  }
}