Upload folder using huggingface_hub

nca_learnable_50_c4/metadata.json

@@ -0,0 +1,72 @@
+{
+  "resume_count": 0,
+  "stages": {
+    "nca_learnable_50_c4": {
+      "resume_count": 0,
+      "initialization_notes": [
+        "Initializing model from base model: /cephfs/user/nklugeco/checkpoints/nca_learnable_50/reset_model for continual pretraining/fine-tuning.",
+        "Applied Liger kernels to the model.",
+        "Number of trainable parameters: 670,127,616",
+        "Shuffling enabled. Shuffling 36 dataset files.",
+        "Collate function will mask token IDs: [0]",
+        "Overriding the number of steps to 10000 as per the `max_steps` argument (check the YAML file if you are not sure).",
+        "Using learning rate decay type: cosine"
+      ],
+      "events": [
+        "Learning rate stage changed to: cosine_decay at step 1000 | nca_learnable_50_c4."
+      ],
+      "run_info": {
+        "Training stage": "nca_learnable_50_c4",
+        "Run ID": "55181",
+        "Hardware": "H200",
+        "World size (total GPUs)": "2",
+        "Precision": "bfloat16"
+      },
+      "Dataset Configuration": {
+        "Num train examples": "2,612,686",
+        "Num validation examples": "8,192",
+        "Length of train dataloader": "81,647",
+        "Max position embeddings (seq length)": "4,096",
+        "Shuffle dataset": "True",
+        "Masked token IDs": "[0]"
+      },
+      "Batch Configuration": {
+        "Num Epochs": "1",
+        "Micro batch size per device": "16",
+        "Gradient accumulation steps": "4",
+        "Total batch size (samples)": "128",
+        "Total batch size (tokens)": "524,288",
+        "Total optimization steps": "10,000",
+        "Steps per epoch": "20,412",
+        "Checkpointing every": "5000 steps"
+      },
+      "Model Architecture": {
+        "Model config": "/cephfs/user/nklugeco/checkpoints/nca_learnable_50/reset_model",
+        "Attention implementation": "flash_attention_4",
+        "Gradient checkpointing": "False",
+        "Liger kernel": "True",
+        "Torch compile": "False",
+        "Trainable parameters": "670,127,616"
+      },
+      "Optimizer Configuration (AdamW)": {
+        "Optimizer type": "adamw",
+        "Max learning rate (Adam)": "0.0005",
+        "Min learning rate": "0.0",
+        "LR scheduler type": "COSINE",
+        "LR decay iterations coef": "0.1",
+        "Warmup steps": "1000",
+        "Weight decay": "0.1",
+        "Beta1": "0.9",
+        "Beta2": "0.95",
+        "Epsilon": "1e-08",
+        "Max grad norm": "1.0"
+      }
+    }
+  },
+  "emissions": {
+    "duration_hours": 7.8614,
+    "energy_consumed_kwh": 4.202192,
+    "co2_emissions_kgco2eq": 1.600825,
+    "num_nodes": 1
+  }
+}

nca_learnable_50_c4/model/config.json

@@ -0,0 +1,34 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "bfloat16",
+  "eos_token_id": 0,
+  "head_dim": 96,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "is_llama_config": true,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_interleaved": false,
+  "rope_parameters": {
+    "rope_theta": 50000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.6.2",
+  "use_cache": false,
+  "vocab_size": 49152
+}

nca_learnable_50_c4/model/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.6.2",
+  "use_cache": false
+}

nca_learnable_50_c4/model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46062c0e9556b22177cbb09563632e338e791ab7e4c744278f646bb7818f60f3
+size 1340284304

nca_learnable_50_c4/model/tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|endoftext|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<repo_name>",
+    "<reponame>",
+    "<file_sep>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 4096,
+  "pad_token": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

nca_learnable_50_c4/validation.jsonl

@@ -0,0 +1,3 @@
+{"kwh": 0.428512, "loss": 3.993336, "stage_name": "nca_learnable_50_c4", "status": "validation", "step": 1000, "val_time_s": 58.295862}
+{"kwh": 2.118686, "loss": 3.051417, "stage_name": "nca_learnable_50_c4", "status": "validation", "step": 5000, "val_time_s": 55.783209}
+{"kwh": 4.200796, "loss": 2.92629, "stage_name": "nca_learnable_50_c4", "status": "validation", "step": 10000, "val_time_s": 55.702015}