DannyAI commited on
Commit
e32356a
·
verified ·
1 Parent(s): 923d602

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +5 -0
  2. README.md +160 -3
  3. adapter_config.json +43 -0
  4. adapter_model.safetensors +3 -0
  5. added_tokens.json +12 -0
  6. chat_template.jinja +1 -0
  7. checkpoint-400/README.md +208 -0
  8. checkpoint-400/adapter_config.json +43 -0
  9. checkpoint-400/adapter_model.safetensors +3 -0
  10. checkpoint-400/added_tokens.json +12 -0
  11. checkpoint-400/chat_template.jinja +1 -0
  12. checkpoint-400/global_step400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +3 -0
  13. checkpoint-400/global_step400/mp_rank_00_model_states.pt +3 -0
  14. checkpoint-400/latest +1 -0
  15. checkpoint-400/merges.txt +0 -0
  16. checkpoint-400/rng_state.pth +3 -0
  17. checkpoint-400/scheduler.pt +3 -0
  18. checkpoint-400/special_tokens_map.json +30 -0
  19. checkpoint-400/tokenizer.json +3 -0
  20. checkpoint-400/tokenizer_config.json +111 -0
  21. checkpoint-400/tokens_state. +1 -0
  22. checkpoint-400/trainer_state.json +1262 -0
  23. checkpoint-400/training_args.bin +3 -0
  24. checkpoint-400/vocab.json +0 -0
  25. checkpoint-400/zero_to_fp32.py +760 -0
  26. checkpoint-500/README.md +208 -0
  27. checkpoint-500/adapter_config.json +43 -0
  28. checkpoint-500/adapter_model.safetensors +3 -0
  29. checkpoint-500/added_tokens.json +12 -0
  30. checkpoint-500/chat_template.jinja +1 -0
  31. checkpoint-500/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +3 -0
  32. checkpoint-500/global_step500/mp_rank_00_model_states.pt +3 -0
  33. checkpoint-500/latest +1 -0
  34. checkpoint-500/merges.txt +0 -0
  35. checkpoint-500/rng_state.pth +3 -0
  36. checkpoint-500/scheduler.pt +3 -0
  37. checkpoint-500/special_tokens_map.json +30 -0
  38. checkpoint-500/tokenizer.json +3 -0
  39. checkpoint-500/tokenizer_config.json +111 -0
  40. checkpoint-500/tokens_state. +1 -0
  41. checkpoint-500/trainer_state.json +1566 -0
  42. checkpoint-500/training_args.bin +3 -0
  43. checkpoint-500/vocab.json +0 -0
  44. checkpoint-500/zero_to_fp32.py +760 -0
  45. checkpoint-600/README.md +208 -0
  46. checkpoint-600/adapter_config.json +43 -0
  47. checkpoint-600/adapter_model.safetensors +3 -0
  48. checkpoint-600/added_tokens.json +12 -0
  49. checkpoint-600/chat_template.jinja +1 -0
  50. checkpoint-600/global_step600/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +3 -0
.gitattributes CHANGED
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ checkpoint-400/tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
38
+ checkpoint-600/tokenizer.json filter=lfs diff=lfs merge=lfs -text
39
+ checkpoint-650/tokenizer.json filter=lfs diff=lfs merge=lfs -text
40
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,3 +1,160 @@
1
- ---
2
- license: mit
3
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ license: mit
4
+ base_model: microsoft/Phi-4-mini-instruct
5
+ tags:
6
+ - axolotl
7
+ - base_model:adapter:microsoft/Phi-4-mini-instruct
8
+ - lora
9
+ - transformers
10
+ datasets:
11
+ - DannyAI/African-History-QA-Dataset
12
+ pipeline_tag: text-generation
13
+ model-index:
14
+ - name: phi4_african_history_lora_ds2_axolotl
15
+ results: []
16
+ ---
17
+
18
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
19
+ should probably proofread and complete it, then remove this comment. -->
20
+
21
+ [<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
22
+ <details><summary>See axolotl config</summary>
23
+
24
+ axolotl version: `0.14.0.dev0`
25
+ ```yaml
26
+ base_model: microsoft/Phi-4-mini-instruct
27
+ model_type: AutoModelForCausalLM
28
+ tokenizer_type: AutoTokenizer
29
+
30
+ # 1. Dataset Configuration
31
+ datasets:
32
+ - path: DannyAI/African-History-QA-Dataset
33
+ split: train
34
+ type: alpaca_chat.load_qa
35
+ system_prompt: "You are a helpful AI assistant specialised in African history."
36
+ test_datasets:
37
+ - path: DannyAI/African-History-QA-Dataset
38
+ split: validation
39
+ type: alpaca_chat.load_qa
40
+ system_prompt: "You are a helpful AI assistant specialised in African history."
41
+
42
+ # 2. Chat Template & Masking
43
+ chat_template: tokenizer_default
44
+ train_on_inputs: false
45
+
46
+ # 3. Batch Size Configuration (REQUIRED TO FIX YOUR ERROR)
47
+ micro_batch_size: 2
48
+ gradient_accumulation_steps: 4
49
+ # Axolotl will now calculate: total_batch_size = 2 * 4 * 1 GPU = 8
50
+
51
+ # 4. LoRA Configuration
52
+ adapter: lora
53
+ lora_r: 8
54
+ lora_alpha: 16
55
+ lora_dropout: 0.05
56
+ lora_target_modules: [q_proj, v_proj, k_proj, o_proj]
57
+
58
+ # 5. Hardware & Efficiency
59
+ sequence_len: 2048
60
+ sample_packing: true
61
+ eval_sample_packing: false
62
+ pad_to_sequence_len: true
63
+ bf16: true
64
+ fp16: false
65
+
66
+ # 6. Training Duration
67
+ max_steps: 650
68
+ # removed
69
+ # num_epochs:
70
+ warmup_steps: 20
71
+ learning_rate: 0.00002
72
+ optimizer: adamw_torch
73
+ lr_scheduler: cosine
74
+
75
+ # 7. Logging & DeepSpeed
76
+ deepspeed: using_axolotl/ds_config_2.json
77
+ wandb_project: phi4_african_history
78
+ wandb_name: phi4_axolotl_stage2
79
+
80
+ eval_strategy: steps
81
+ eval_steps: 50
82
+ save_strategy: steps
83
+ save_steps: 100
84
+ logging_steps: 5
85
+
86
+ # 8. Public Hugging Face Hub Upload
87
+ hub_model_id: DannyAI/phi4_african_history_lora_ds2_axolotl
88
+ push_adapter_to_hub: true
89
+ hub_private_repo: false
90
+
91
+ ```
92
+
93
+ </details><br>
94
+
95
+ # phi4_african_history_lora_ds2_axolotl
96
+
97
+ This model is a fine-tuned version of [microsoft/Phi-4-mini-instruct](https://huggingface.co/microsoft/Phi-4-mini-instruct) on the DannyAI/African-History-QA-Dataset dataset.
98
+ It achieves the following results on the evaluation set:
99
+ - Loss: 1.7608
100
+ - Ppl: 5.8168
101
+ - Memory/max Active (gib): 14.82
102
+ - Memory/max Allocated (gib): 14.82
103
+ - Memory/device Reserved (gib): 31.82
104
+
105
+ ## Model description
106
+
107
+ More information needed
108
+
109
+ ## Intended uses & limitations
110
+
111
+ More information needed
112
+
113
+ ## Training and evaluation data
114
+
115
+ More information needed
116
+
117
+ ## Training procedure
118
+
119
+ ### Training hyperparameters
120
+
121
+ The following hyperparameters were used during training:
122
+ - learning_rate: 2e-05
123
+ - train_batch_size: 2
124
+ - eval_batch_size: 2
125
+ - seed: 42
126
+ - distributed_type: multi-GPU
127
+ - gradient_accumulation_steps: 4
128
+ - total_train_batch_size: 8
129
+ - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
130
+ - lr_scheduler_type: cosine
131
+ - lr_scheduler_warmup_steps: 20
132
+ - training_steps: 650
133
+
134
+ ### Training results
135
+
136
+ | Training Loss | Epoch | Step | Validation Loss | Ppl | Active (gib) | Allocated (gib) | Reserved (gib) |
137
+ |:-------------:|:-------:|:----:|:---------------:|:------:|:------------:|:---------------:|:--------------:|
138
+ | No log | 0 | 0 | 2.1261 | 8.3822 | 14.81 | 14.81 | 15.32 |
139
+ | 5.5167 | 3.8627 | 50 | 2.1056 | 8.2118 | 14.82 | 14.82 | 31.8 |
140
+ | 4.5059 | 7.7059 | 100 | 2.0382 | 7.6764 | 14.82 | 14.82 | 31.82 |
141
+ | 3.8251 | 11.5490 | 150 | 1.9809 | 7.2491 | 14.82 | 14.82 | 31.82 |
142
+ | 3.4152 | 15.3922 | 200 | 1.9343 | 6.9193 | 14.82 | 14.82 | 31.82 |
143
+ | 3.1617 | 19.2353 | 250 | 1.8731 | 6.5085 | 14.82 | 14.82 | 31.82 |
144
+ | 2.9075 | 23.0784 | 300 | 1.8246 | 6.2002 | 14.82 | 14.82 | 31.82 |
145
+ | 2.8267 | 26.9412 | 350 | 1.7945 | 6.0164 | 14.82 | 14.82 | 31.82 |
146
+ | 2.7239 | 30.7843 | 400 | 1.7794 | 5.9262 | 14.82 | 14.82 | 31.82 |
147
+ | 2.7275 | 34.6275 | 450 | 1.7697 | 5.8690 | 14.82 | 14.82 | 31.82 |
148
+ | 2.6912 | 38.4706 | 500 | 1.7634 | 5.8325 | 14.82 | 14.82 | 31.82 |
149
+ | 2.6632 | 42.3137 | 550 | 1.7618 | 5.8227 | 14.82 | 14.82 | 31.82 |
150
+ | 2.6604 | 46.1569 | 600 | 1.7609 | 5.8179 | 14.82 | 14.82 | 31.82 |
151
+ | 2.6795 | 50.0 | 650 | 1.7608 | 5.8168 | 14.82 | 14.82 | 31.82 |
152
+
153
+
154
+ ### Framework versions
155
+
156
+ - PEFT 0.18.1
157
+ - Transformers 4.57.6
158
+ - Pytorch 2.9.1+cu128
159
+ - Datasets 4.5.0
160
+ - Tokenizers 0.22.2
adapter_config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "microsoft/Phi-4-mini-instruct",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": null,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 16,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.05,
22
+ "megatron_config": null,
23
+ "megatron_core": "megatron.core",
24
+ "modules_to_save": null,
25
+ "peft_type": "LORA",
26
+ "peft_version": "0.18.1",
27
+ "qalora_group_size": 16,
28
+ "r": 8,
29
+ "rank_pattern": {},
30
+ "revision": null,
31
+ "target_modules": [
32
+ "k_proj",
33
+ "q_proj",
34
+ "v_proj",
35
+ "o_proj"
36
+ ],
37
+ "target_parameters": [],
38
+ "task_type": "CAUSAL_LM",
39
+ "trainable_token_indices": null,
40
+ "use_dora": false,
41
+ "use_qalora": false,
42
+ "use_rslora": false
43
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6145644d56e0da046f760e2c55620e7d4aff3c5f1e96230daddd160bb94275e8
3
+ size 3154216
added_tokens.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|/tool_call|>": 200026,
3
+ "<|/tool|>": 200024,
4
+ "<|assistant|>": 200019,
5
+ "<|end|>": 200020,
6
+ "<|system|>": 200022,
7
+ "<|tag|>": 200028,
8
+ "<|tool_call|>": 200025,
9
+ "<|tool_response|>": 200027,
10
+ "<|tool|>": 200023,
11
+ "<|user|>": 200021
12
+ }
chat_template.jinja ADDED
@@ -0,0 +1 @@
 
 
1
+ {% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}
checkpoint-400/README.md ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: microsoft/Phi-4-mini-instruct
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - axolotl
7
+ - base_model:adapter:microsoft/Phi-4-mini-instruct
8
+ - lora
9
+ - transformers
10
+ ---
11
+
12
+ # Model Card for Model ID
13
+
14
+ <!-- Provide a quick summary of what the model is/does. -->
15
+
16
+
17
+
18
+ ## Model Details
19
+
20
+ ### Model Description
21
+
22
+ <!-- Provide a longer summary of what this model is. -->
23
+
24
+
25
+
26
+ - **Developed by:** [More Information Needed]
27
+ - **Funded by [optional]:** [More Information Needed]
28
+ - **Shared by [optional]:** [More Information Needed]
29
+ - **Model type:** [More Information Needed]
30
+ - **Language(s) (NLP):** [More Information Needed]
31
+ - **License:** [More Information Needed]
32
+ - **Finetuned from model [optional]:** [More Information Needed]
33
+
34
+ ### Model Sources [optional]
35
+
36
+ <!-- Provide the basic links for the model. -->
37
+
38
+ - **Repository:** [More Information Needed]
39
+ - **Paper [optional]:** [More Information Needed]
40
+ - **Demo [optional]:** [More Information Needed]
41
+
42
+ ## Uses
43
+
44
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
45
+
46
+ ### Direct Use
47
+
48
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
49
+
50
+ [More Information Needed]
51
+
52
+ ### Downstream Use [optional]
53
+
54
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
55
+
56
+ [More Information Needed]
57
+
58
+ ### Out-of-Scope Use
59
+
60
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
61
+
62
+ [More Information Needed]
63
+
64
+ ## Bias, Risks, and Limitations
65
+
66
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
67
+
68
+ [More Information Needed]
69
+
70
+ ### Recommendations
71
+
72
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
73
+
74
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
75
+
76
+ ## How to Get Started with the Model
77
+
78
+ Use the code below to get started with the model.
79
+
80
+ [More Information Needed]
81
+
82
+ ## Training Details
83
+
84
+ ### Training Data
85
+
86
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
87
+
88
+ [More Information Needed]
89
+
90
+ ### Training Procedure
91
+
92
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
93
+
94
+ #### Preprocessing [optional]
95
+
96
+ [More Information Needed]
97
+
98
+
99
+ #### Training Hyperparameters
100
+
101
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
102
+
103
+ #### Speeds, Sizes, Times [optional]
104
+
105
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
106
+
107
+ [More Information Needed]
108
+
109
+ ## Evaluation
110
+
111
+ <!-- This section describes the evaluation protocols and provides the results. -->
112
+
113
+ ### Testing Data, Factors & Metrics
114
+
115
+ #### Testing Data
116
+
117
+ <!-- This should link to a Dataset Card if possible. -->
118
+
119
+ [More Information Needed]
120
+
121
+ #### Factors
122
+
123
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
124
+
125
+ [More Information Needed]
126
+
127
+ #### Metrics
128
+
129
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
130
+
131
+ [More Information Needed]
132
+
133
+ ### Results
134
+
135
+ [More Information Needed]
136
+
137
+ #### Summary
138
+
139
+
140
+
141
+ ## Model Examination [optional]
142
+
143
+ <!-- Relevant interpretability work for the model goes here -->
144
+
145
+ [More Information Needed]
146
+
147
+ ## Environmental Impact
148
+
149
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
150
+
151
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
152
+
153
+ - **Hardware Type:** [More Information Needed]
154
+ - **Hours used:** [More Information Needed]
155
+ - **Cloud Provider:** [More Information Needed]
156
+ - **Compute Region:** [More Information Needed]
157
+ - **Carbon Emitted:** [More Information Needed]
158
+
159
+ ## Technical Specifications [optional]
160
+
161
+ ### Model Architecture and Objective
162
+
163
+ [More Information Needed]
164
+
165
+ ### Compute Infrastructure
166
+
167
+ [More Information Needed]
168
+
169
+ #### Hardware
170
+
171
+ [More Information Needed]
172
+
173
+ #### Software
174
+
175
+ [More Information Needed]
176
+
177
+ ## Citation [optional]
178
+
179
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
180
+
181
+ **BibTeX:**
182
+
183
+ [More Information Needed]
184
+
185
+ **APA:**
186
+
187
+ [More Information Needed]
188
+
189
+ ## Glossary [optional]
190
+
191
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
192
+
193
+ [More Information Needed]
194
+
195
+ ## More Information [optional]
196
+
197
+ [More Information Needed]
198
+
199
+ ## Model Card Authors [optional]
200
+
201
+ [More Information Needed]
202
+
203
+ ## Model Card Contact
204
+
205
+ [More Information Needed]
206
+ ### Framework versions
207
+
208
+ - PEFT 0.18.1
checkpoint-400/adapter_config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "microsoft/Phi-4-mini-instruct",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": null,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 16,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.05,
22
+ "megatron_config": null,
23
+ "megatron_core": "megatron.core",
24
+ "modules_to_save": null,
25
+ "peft_type": "LORA",
26
+ "peft_version": "0.18.1",
27
+ "qalora_group_size": 16,
28
+ "r": 8,
29
+ "rank_pattern": {},
30
+ "revision": null,
31
+ "target_modules": [
32
+ "k_proj",
33
+ "q_proj",
34
+ "v_proj",
35
+ "o_proj"
36
+ ],
37
+ "target_parameters": [],
38
+ "task_type": "CAUSAL_LM",
39
+ "trainable_token_indices": null,
40
+ "use_dora": false,
41
+ "use_qalora": false,
42
+ "use_rslora": false
43
+ }
checkpoint-400/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7cbd9cc00bd9a7d94037119486a80e1dc4e072b3f788a812d8d487f6f691c970
3
+ size 3154216
checkpoint-400/added_tokens.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|/tool_call|>": 200026,
3
+ "<|/tool|>": 200024,
4
+ "<|assistant|>": 200019,
5
+ "<|end|>": 200020,
6
+ "<|system|>": 200022,
7
+ "<|tag|>": 200028,
8
+ "<|tool_call|>": 200025,
9
+ "<|tool_response|>": 200027,
10
+ "<|tool|>": 200023,
11
+ "<|user|>": 200021
12
+ }
checkpoint-400/chat_template.jinja ADDED
@@ -0,0 +1 @@
 
 
1
+ {% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}
checkpoint-400/global_step400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6eab0e6f36d51af66f524eba52e23f34ac66c447cd5e83293881b93e4b6f5f34
3
+ size 18884901
checkpoint-400/global_step400/mp_rank_00_model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:395b136517f4d0a2b7166240bc6356550a9482bfa55e2d102f5319ba17d7193f
3
+ size 1232414501
checkpoint-400/latest ADDED
@@ -0,0 +1 @@
 
 
1
+ global_step400
checkpoint-400/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-400/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:39b21669dff56c45be193db845f9256d767dd1fd4ad4f7cdd537c4ff40acef24
3
+ size 14709
checkpoint-400/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3ba2cb63a914a490682d757834ae6e0b0262b7147a90880edabed28124ba786
3
+ size 1465
checkpoint-400/special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|endoftext|>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<|endoftext|>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<|endoftext|>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
checkpoint-400/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:382cc235b56c725945e149cc25f191da667c836655efd0857b004320e90e91ea
3
+ size 15524095
checkpoint-400/tokenizer_config.json ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": false,
5
+ "added_tokens_decoder": {
6
+ "199999": {
7
+ "content": "<|endoftext|>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "200018": {
15
+ "content": "<|endofprompt|>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "200019": {
23
+ "content": "<|assistant|>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": true,
27
+ "single_word": false,
28
+ "special": true
29
+ },
30
+ "200020": {
31
+ "content": "<|end|>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": true,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
+ "200021": {
39
+ "content": "<|user|>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": true,
43
+ "single_word": false,
44
+ "special": true
45
+ },
46
+ "200022": {
47
+ "content": "<|system|>",
48
+ "lstrip": false,
49
+ "normalized": false,
50
+ "rstrip": true,
51
+ "single_word": false,
52
+ "special": true
53
+ },
54
+ "200023": {
55
+ "content": "<|tool|>",
56
+ "lstrip": false,
57
+ "normalized": false,
58
+ "rstrip": true,
59
+ "single_word": false,
60
+ "special": false
61
+ },
62
+ "200024": {
63
+ "content": "<|/tool|>",
64
+ "lstrip": false,
65
+ "normalized": false,
66
+ "rstrip": true,
67
+ "single_word": false,
68
+ "special": false
69
+ },
70
+ "200025": {
71
+ "content": "<|tool_call|>",
72
+ "lstrip": false,
73
+ "normalized": false,
74
+ "rstrip": true,
75
+ "single_word": false,
76
+ "special": false
77
+ },
78
+ "200026": {
79
+ "content": "<|/tool_call|>",
80
+ "lstrip": false,
81
+ "normalized": false,
82
+ "rstrip": true,
83
+ "single_word": false,
84
+ "special": false
85
+ },
86
+ "200027": {
87
+ "content": "<|tool_response|>",
88
+ "lstrip": false,
89
+ "normalized": false,
90
+ "rstrip": true,
91
+ "single_word": false,
92
+ "special": false
93
+ },
94
+ "200028": {
95
+ "content": "<|tag|>",
96
+ "lstrip": false,
97
+ "normalized": false,
98
+ "rstrip": true,
99
+ "single_word": false,
100
+ "special": true
101
+ }
102
+ },
103
+ "bos_token": "<|endoftext|>",
104
+ "clean_up_tokenization_spaces": false,
105
+ "eos_token": "<|endoftext|>",
106
+ "extra_special_tokens": {},
107
+ "model_max_length": 131072,
108
+ "pad_token": "<|endoftext|>",
109
+ "tokenizer_class": "GPT2Tokenizer",
110
+ "unk_token": "<|endoftext|>"
111
+ }
checkpoint-400/tokens_state. ADDED
@@ -0,0 +1 @@
 
 
1
+ {"total": 6430720, "trainable": 2909069}
checkpoint-400/trainer_state.json ADDED
@@ -0,0 +1,1262 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 30.784313725490197,
6
+ "eval_steps": 50,
7
+ "global_step": 400,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0,
14
+ "eval_loss": 2.1261069774627686,
15
+ "eval_ppl": 8.38217,
16
+ "eval_runtime": 51.9298,
17
+ "eval_samples_per_second": 3.851,
18
+ "eval_steps_per_second": 1.926,
19
+ "memory/device_reserved (GiB)": 15.32,
20
+ "memory/max_active (GiB)": 14.81,
21
+ "memory/max_allocated (GiB)": 14.81,
22
+ "step": 0
23
+ },
24
+ {
25
+ "epoch": 0.39215686274509803,
26
+ "grad_norm": 0.27175864577293396,
27
+ "learning_rate": 4.000000000000001e-06,
28
+ "loss": 6.0524,
29
+ "memory/device_reserved (GiB)": 31.8,
30
+ "memory/max_active (GiB)": 30.03,
31
+ "memory/max_allocated (GiB)": 30.03,
32
+ "ppl": 425.13212,
33
+ "step": 5,
34
+ "tokens/total": 81920,
35
+ "tokens/train_per_sec_per_gpu": 46.17,
36
+ "tokens/trainable": 36899
37
+ },
38
+ {
39
+ "epoch": 0.7843137254901961,
40
+ "grad_norm": 0.26692718267440796,
41
+ "learning_rate": 9e-06,
42
+ "loss": 5.992,
43
+ "memory/device_reserved (GiB)": 31.8,
44
+ "memory/max_active (GiB)": 30.03,
45
+ "memory/max_allocated (GiB)": 30.03,
46
+ "ppl": 400.21424,
47
+ "step": 10,
48
+ "tokens/total": 163840,
49
+ "tokens/train_per_sec_per_gpu": 47.3,
50
+ "tokens/trainable": 74260
51
+ },
52
+ {
53
+ "epoch": 1.156862745098039,
54
+ "grad_norm": 0.306497722864151,
55
+ "learning_rate": 1.4e-05,
56
+ "loss": 5.9906,
57
+ "memory/device_reserved (GiB)": 31.8,
58
+ "memory/max_active (GiB)": 30.03,
59
+ "memory/max_allocated (GiB)": 30.03,
60
+ "ppl": 399.65433,
61
+ "step": 15,
62
+ "tokens/total": 241664,
63
+ "tokens/train_per_sec_per_gpu": 48.66,
64
+ "tokens/trainable": 109275
65
+ },
66
+ {
67
+ "epoch": 1.5490196078431373,
68
+ "grad_norm": 0.3827843964099884,
69
+ "learning_rate": 1.9e-05,
70
+ "loss": 6.0295,
71
+ "memory/device_reserved (GiB)": 31.8,
72
+ "memory/max_active (GiB)": 30.03,
73
+ "memory/max_allocated (GiB)": 30.03,
74
+ "ppl": 415.50722,
75
+ "step": 20,
76
+ "tokens/total": 323584,
77
+ "tokens/train_per_sec_per_gpu": 46.01,
78
+ "tokens/trainable": 146594
79
+ },
80
+ {
81
+ "epoch": 1.9411764705882353,
82
+ "grad_norm": 0.46802398562431335,
83
+ "learning_rate": 1.9998010727705237e-05,
84
+ "loss": 5.9689,
85
+ "memory/device_reserved (GiB)": 31.8,
86
+ "memory/max_active (GiB)": 30.03,
87
+ "memory/max_allocated (GiB)": 30.03,
88
+ "ppl": 391.07525,
89
+ "step": 25,
90
+ "tokens/total": 405504,
91
+ "tokens/train_per_sec_per_gpu": 45.66,
92
+ "tokens/trainable": 183790
93
+ },
94
+ {
95
+ "epoch": 2.313725490196078,
96
+ "grad_norm": 0.5154851675033569,
97
+ "learning_rate": 1.9989930665413148e-05,
98
+ "loss": 5.8882,
99
+ "memory/device_reserved (GiB)": 31.8,
100
+ "memory/max_active (GiB)": 30.03,
101
+ "memory/max_allocated (GiB)": 30.03,
102
+ "ppl": 360.75534,
103
+ "step": 30,
104
+ "tokens/total": 483328,
105
+ "tokens/train_per_sec_per_gpu": 45.15,
106
+ "tokens/trainable": 218443
107
+ },
108
+ {
109
+ "epoch": 2.7058823529411766,
110
+ "grad_norm": 0.6076357364654541,
111
+ "learning_rate": 1.9975640502598243e-05,
112
+ "loss": 5.7916,
113
+ "memory/device_reserved (GiB)": 31.8,
114
+ "memory/max_active (GiB)": 30.03,
115
+ "memory/max_allocated (GiB)": 30.03,
116
+ "ppl": 327.53666,
117
+ "step": 35,
118
+ "tokens/total": 565248,
119
+ "tokens/train_per_sec_per_gpu": 47.92,
120
+ "tokens/trainable": 255824
121
+ },
122
+ {
123
+ "epoch": 3.0784313725490198,
124
+ "grad_norm": 0.6261164546012878,
125
+ "learning_rate": 1.995514912254015e-05,
126
+ "loss": 5.7018,
127
+ "memory/device_reserved (GiB)": 31.8,
128
+ "memory/max_active (GiB)": 30.03,
129
+ "memory/max_allocated (GiB)": 30.03,
130
+ "ppl": 299.40585,
131
+ "step": 40,
132
+ "tokens/total": 643072,
133
+ "tokens/train_per_sec_per_gpu": 45.74,
134
+ "tokens/trainable": 290916
135
+ },
136
+ {
137
+ "epoch": 3.4705882352941178,
138
+ "grad_norm": 0.7209100127220154,
139
+ "learning_rate": 1.9928469263418376e-05,
140
+ "loss": 5.5957,
141
+ "memory/device_reserved (GiB)": 31.8,
142
+ "memory/max_active (GiB)": 30.03,
143
+ "memory/max_allocated (GiB)": 30.03,
144
+ "ppl": 269.26607,
145
+ "step": 45,
146
+ "tokens/total": 724992,
147
+ "tokens/train_per_sec_per_gpu": 46.62,
148
+ "tokens/trainable": 328157
149
+ },
150
+ {
151
+ "epoch": 3.8627450980392157,
152
+ "grad_norm": 0.6933228373527527,
153
+ "learning_rate": 1.9895617510393773e-05,
154
+ "loss": 5.5167,
155
+ "memory/device_reserved (GiB)": 31.8,
156
+ "memory/max_active (GiB)": 30.03,
157
+ "memory/max_allocated (GiB)": 30.03,
158
+ "ppl": 248.8126,
159
+ "step": 50,
160
+ "tokens/total": 806912,
161
+ "tokens/train_per_sec_per_gpu": 45.37,
162
+ "tokens/trainable": 365292
163
+ },
164
+ {
165
+ "epoch": 3.8627450980392157,
166
+ "eval_loss": 2.1055686473846436,
167
+ "eval_ppl": 8.21177,
168
+ "eval_runtime": 52.5301,
169
+ "eval_samples_per_second": 3.807,
170
+ "eval_steps_per_second": 1.904,
171
+ "memory/device_reserved (GiB)": 31.8,
172
+ "memory/max_active (GiB)": 14.82,
173
+ "memory/max_allocated (GiB)": 14.82,
174
+ "step": 50
175
+ },
176
+ {
177
+ "epoch": 4.235294117647059,
178
+ "grad_norm": 0.7358637452125549,
179
+ "learning_rate": 1.985661428529863e-05,
180
+ "loss": 5.343,
181
+ "memory/device_reserved (GiB)": 31.82,
182
+ "memory/max_active (GiB)": 30.03,
183
+ "memory/max_allocated (GiB)": 30.03,
184
+ "ppl": 209.13919,
185
+ "step": 55,
186
+ "tokens/total": 884736,
187
+ "tokens/train_per_sec_per_gpu": 44.66,
188
+ "tokens/trainable": 399785
189
+ },
190
+ {
191
+ "epoch": 4.627450980392156,
192
+ "grad_norm": 0.6546111702919006,
193
+ "learning_rate": 1.9811483833941726e-05,
194
+ "loss": 5.2353,
195
+ "memory/device_reserved (GiB)": 31.82,
196
+ "memory/max_active (GiB)": 30.03,
197
+ "memory/max_allocated (GiB)": 30.03,
198
+ "ppl": 187.78543,
199
+ "step": 60,
200
+ "tokens/total": 966656,
201
+ "tokens/train_per_sec_per_gpu": 42.79,
202
+ "tokens/trainable": 437478
203
+ },
204
+ {
205
+ "epoch": 5.0,
206
+ "grad_norm": 0.7404906749725342,
207
+ "learning_rate": 1.9760254211036245e-05,
208
+ "loss": 5.1051,
209
+ "memory/device_reserved (GiB)": 31.82,
210
+ "memory/max_active (GiB)": 30.03,
211
+ "memory/max_allocated (GiB)": 30.03,
212
+ "ppl": 164.86056,
213
+ "step": 65,
214
+ "tokens/total": 1044480,
215
+ "tokens/train_per_sec_per_gpu": 41.51,
216
+ "tokens/trainable": 472345
217
+ },
218
+ {
219
+ "epoch": 5.392156862745098,
220
+ "grad_norm": 0.6113602519035339,
221
+ "learning_rate": 1.9702957262759964e-05,
222
+ "loss": 4.9746,
223
+ "memory/device_reserved (GiB)": 31.82,
224
+ "memory/max_active (GiB)": 30.03,
225
+ "memory/max_allocated (GiB)": 30.03,
226
+ "ppl": 144.69094,
227
+ "step": 70,
228
+ "tokens/total": 1126400,
229
+ "tokens/train_per_sec_per_gpu": 47.33,
230
+ "tokens/trainable": 509900
231
+ },
232
+ {
233
+ "epoch": 5.784313725490196,
234
+ "grad_norm": 0.5721343755722046,
235
+ "learning_rate": 1.9639628606958535e-05,
236
+ "loss": 4.8985,
237
+ "memory/device_reserved (GiB)": 31.82,
238
+ "memory/max_active (GiB)": 30.03,
239
+ "memory/max_allocated (GiB)": 30.03,
240
+ "ppl": 134.0885,
241
+ "step": 75,
242
+ "tokens/total": 1208320,
243
+ "tokens/train_per_sec_per_gpu": 46.45,
244
+ "tokens/trainable": 547077
245
+ },
246
+ {
247
+ "epoch": 6.1568627450980395,
248
+ "grad_norm": 0.6451320052146912,
249
+ "learning_rate": 1.9570307611004124e-05,
250
+ "loss": 4.7955,
251
+ "memory/device_reserved (GiB)": 31.82,
252
+ "memory/max_active (GiB)": 30.03,
253
+ "memory/max_allocated (GiB)": 30.03,
254
+ "ppl": 120.96485,
255
+ "step": 80,
256
+ "tokens/total": 1286144,
257
+ "tokens/train_per_sec_per_gpu": 47.54,
258
+ "tokens/trainable": 581824
259
+ },
260
+ {
261
+ "epoch": 6.549019607843137,
262
+ "grad_norm": 0.6520641446113586,
263
+ "learning_rate": 1.9495037367323264e-05,
264
+ "loss": 4.7514,
265
+ "memory/device_reserved (GiB)": 31.82,
266
+ "memory/max_active (GiB)": 30.03,
267
+ "memory/max_allocated (GiB)": 30.03,
268
+ "ppl": 115.74622,
269
+ "step": 85,
270
+ "tokens/total": 1368064,
271
+ "tokens/train_per_sec_per_gpu": 44.41,
272
+ "tokens/trainable": 618867
273
+ },
274
+ {
275
+ "epoch": 6.9411764705882355,
276
+ "grad_norm": 0.6384134888648987,
277
+ "learning_rate": 1.9413864666609036e-05,
278
+ "loss": 4.6298,
279
+ "memory/device_reserved (GiB)": 31.82,
280
+ "memory/max_active (GiB)": 30.03,
281
+ "memory/max_allocated (GiB)": 30.03,
282
+ "ppl": 102.49356,
283
+ "step": 90,
284
+ "tokens/total": 1449984,
285
+ "tokens/train_per_sec_per_gpu": 44.14,
286
+ "tokens/trainable": 656298
287
+ },
288
+ {
289
+ "epoch": 7.313725490196078,
290
+ "grad_norm": 0.6989919543266296,
291
+ "learning_rate": 1.9326839968734278e-05,
292
+ "loss": 4.5767,
293
+ "memory/device_reserved (GiB)": 31.82,
294
+ "memory/max_active (GiB)": 30.03,
295
+ "memory/max_allocated (GiB)": 30.03,
296
+ "ppl": 97.19313,
297
+ "step": 95,
298
+ "tokens/total": 1527808,
299
+ "tokens/train_per_sec_per_gpu": 45.78,
300
+ "tokens/trainable": 691156
301
+ },
302
+ {
303
+ "epoch": 7.705882352941177,
304
+ "grad_norm": 0.630689263343811,
305
+ "learning_rate": 1.9234017371383946e-05,
306
+ "loss": 4.5059,
307
+ "memory/device_reserved (GiB)": 31.82,
308
+ "memory/max_active (GiB)": 30.03,
309
+ "memory/max_allocated (GiB)": 30.03,
310
+ "ppl": 90.5498,
311
+ "step": 100,
312
+ "tokens/total": 1609728,
313
+ "tokens/train_per_sec_per_gpu": 46.37,
314
+ "tokens/trainable": 727983
315
+ },
316
+ {
317
+ "epoch": 7.705882352941177,
318
+ "eval_loss": 2.0381510257720947,
319
+ "eval_ppl": 7.6764,
320
+ "eval_runtime": 52.3831,
321
+ "eval_samples_per_second": 3.818,
322
+ "eval_steps_per_second": 1.909,
323
+ "memory/device_reserved (GiB)": 31.82,
324
+ "memory/max_active (GiB)": 14.82,
325
+ "memory/max_allocated (GiB)": 14.82,
326
+ "step": 100
327
+ },
328
+ {
329
+ "epoch": 8.07843137254902,
330
+ "grad_norm": 0.6709732413291931,
331
+ "learning_rate": 1.913545457642601e-05,
332
+ "loss": 4.3844,
333
+ "memory/device_reserved (GiB)": 31.82,
334
+ "memory/max_active (GiB)": 30.03,
335
+ "memory/max_allocated (GiB)": 30.03,
336
+ "ppl": 80.19009,
337
+ "step": 105,
338
+ "tokens/total": 1687552,
339
+ "tokens/train_per_sec_per_gpu": 45.03,
340
+ "tokens/trainable": 763205
341
+ },
342
+ {
343
+ "epoch": 8.470588235294118,
344
+ "grad_norm": 0.6814770698547363,
345
+ "learning_rate": 1.903121285404192e-05,
346
+ "loss": 4.3581,
347
+ "memory/device_reserved (GiB)": 31.82,
348
+ "memory/max_active (GiB)": 30.03,
349
+ "memory/max_allocated (GiB)": 30.03,
350
+ "ppl": 78.10859,
351
+ "step": 110,
352
+ "tokens/total": 1769472,
353
+ "tokens/train_per_sec_per_gpu": 45.04,
354
+ "tokens/trainable": 801058
355
+ },
356
+ {
357
+ "epoch": 8.862745098039216,
358
+ "grad_norm": 0.7326321005821228,
359
+ "learning_rate": 1.8921357004638837e-05,
360
+ "loss": 4.2923,
361
+ "memory/device_reserved (GiB)": 31.82,
362
+ "memory/max_active (GiB)": 30.03,
363
+ "memory/max_allocated (GiB)": 30.03,
364
+ "ppl": 73.13448,
365
+ "step": 115,
366
+ "tokens/total": 1851392,
367
+ "tokens/train_per_sec_per_gpu": 44.88,
368
+ "tokens/trainable": 837904
369
+ },
370
+ {
371
+ "epoch": 9.235294117647058,
372
+ "grad_norm": 0.7615563869476318,
373
+ "learning_rate": 1.880595531856738e-05,
374
+ "loss": 4.1865,
375
+ "memory/device_reserved (GiB)": 31.82,
376
+ "memory/max_active (GiB)": 30.03,
377
+ "memory/max_allocated (GiB)": 30.03,
378
+ "ppl": 65.79212,
379
+ "step": 120,
380
+ "tokens/total": 1929216,
381
+ "tokens/train_per_sec_per_gpu": 48.18,
382
+ "tokens/trainable": 872791
383
+ },
384
+ {
385
+ "epoch": 9.627450980392156,
386
+ "grad_norm": 0.6976982951164246,
387
+ "learning_rate": 1.868507953366989e-05,
388
+ "loss": 4.157,
389
+ "memory/device_reserved (GiB)": 31.82,
390
+ "memory/max_active (GiB)": 30.03,
391
+ "memory/max_allocated (GiB)": 30.03,
392
+ "ppl": 63.8796,
393
+ "step": 125,
394
+ "tokens/total": 2011136,
395
+ "tokens/train_per_sec_per_gpu": 48.15,
396
+ "tokens/trainable": 910332
397
+ },
398
+ {
399
+ "epoch": 10.0,
400
+ "grad_norm": 0.9011051058769226,
401
+ "learning_rate": 1.855880479068559e-05,
402
+ "loss": 4.0724,
403
+ "memory/device_reserved (GiB)": 31.82,
404
+ "memory/max_active (GiB)": 30.03,
405
+ "memory/max_allocated (GiB)": 30.03,
406
+ "ppl": 58.69767,
407
+ "step": 130,
408
+ "tokens/total": 2088960,
409
+ "tokens/train_per_sec_per_gpu": 44.4,
410
+ "tokens/trainable": 944690
411
+ },
412
+ {
413
+ "epoch": 10.392156862745098,
414
+ "grad_norm": 0.7039150595664978,
415
+ "learning_rate": 1.8427209586540392e-05,
416
+ "loss": 4.0452,
417
+ "memory/device_reserved (GiB)": 31.82,
418
+ "memory/max_active (GiB)": 30.03,
419
+ "memory/max_allocated (GiB)": 30.03,
420
+ "ppl": 57.12261,
421
+ "step": 135,
422
+ "tokens/total": 2170880,
423
+ "tokens/train_per_sec_per_gpu": 44.67,
424
+ "tokens/trainable": 981807
425
+ },
426
+ {
427
+ "epoch": 10.784313725490197,
428
+ "grad_norm": 0.7844482660293579,
429
+ "learning_rate": 1.8290375725550417e-05,
430
+ "loss": 3.9809,
431
+ "memory/device_reserved (GiB)": 31.82,
432
+ "memory/max_active (GiB)": 30.03,
433
+ "memory/max_allocated (GiB)": 30.03,
434
+ "ppl": 53.56522,
435
+ "step": 140,
436
+ "tokens/total": 2252800,
437
+ "tokens/train_per_sec_per_gpu": 52.01,
438
+ "tokens/trainable": 1019070
439
+ },
440
+ {
441
+ "epoch": 11.156862745098039,
442
+ "grad_norm": 0.8043951392173767,
443
+ "learning_rate": 1.8148388268569453e-05,
444
+ "loss": 3.8289,
445
+ "memory/device_reserved (GiB)": 31.82,
446
+ "memory/max_active (GiB)": 30.03,
447
+ "memory/max_allocated (GiB)": 30.03,
448
+ "ppl": 46.0119,
449
+ "step": 145,
450
+ "tokens/total": 2330624,
451
+ "tokens/train_per_sec_per_gpu": 42.58,
452
+ "tokens/trainable": 1053699
453
+ },
454
+ {
455
+ "epoch": 11.549019607843137,
456
+ "grad_norm": 0.7718515992164612,
457
+ "learning_rate": 1.8001335480112067e-05,
458
+ "loss": 3.8251,
459
+ "memory/device_reserved (GiB)": 31.82,
460
+ "memory/max_active (GiB)": 30.03,
461
+ "memory/max_allocated (GiB)": 30.03,
462
+ "ppl": 45.83738,
463
+ "step": 150,
464
+ "tokens/total": 2412544,
465
+ "tokens/train_per_sec_per_gpu": 45.26,
466
+ "tokens/trainable": 1091200
467
+ },
468
+ {
469
+ "epoch": 11.549019607843137,
470
+ "eval_loss": 1.980879783630371,
471
+ "eval_ppl": 7.24912,
472
+ "eval_runtime": 52.4263,
473
+ "eval_samples_per_second": 3.815,
474
+ "eval_steps_per_second": 1.907,
475
+ "memory/device_reserved (GiB)": 31.82,
476
+ "memory/max_active (GiB)": 14.82,
477
+ "memory/max_allocated (GiB)": 14.82,
478
+ "step": 150
479
+ },
480
+ {
481
+ "epoch": 11.941176470588236,
482
+ "grad_norm": 0.8142716288566589,
483
+ "learning_rate": 1.7849308773485226e-05,
484
+ "loss": 3.7862,
485
+ "memory/device_reserved (GiB)": 31.82,
486
+ "memory/max_active (GiB)": 30.03,
487
+ "memory/max_allocated (GiB)": 30.03,
488
+ "ppl": 44.08855,
489
+ "step": 155,
490
+ "tokens/total": 2494464,
491
+ "tokens/train_per_sec_per_gpu": 46.91,
492
+ "tokens/trainable": 1128496
493
+ },
494
+ {
495
+ "epoch": 12.313725490196079,
496
+ "grad_norm": 0.7591995596885681,
497
+ "learning_rate": 1.769240265396249e-05,
498
+ "loss": 3.7156,
499
+ "memory/device_reserved (GiB)": 31.82,
500
+ "memory/max_active (GiB)": 30.03,
501
+ "memory/max_allocated (GiB)": 30.03,
502
+ "ppl": 41.08323,
503
+ "step": 160,
504
+ "tokens/total": 2572288,
505
+ "tokens/train_per_sec_per_gpu": 45.85,
506
+ "tokens/trainable": 1164008
507
+ },
508
+ {
509
+ "epoch": 12.705882352941176,
510
+ "grad_norm": 0.7124404311180115,
511
+ "learning_rate": 1.7530714660036112e-05,
512
+ "loss": 3.6955,
513
+ "memory/device_reserved (GiB)": 31.82,
514
+ "memory/max_active (GiB)": 30.03,
515
+ "memory/max_allocated (GiB)": 30.03,
516
+ "ppl": 40.2657,
517
+ "step": 165,
518
+ "tokens/total": 2654208,
519
+ "tokens/train_per_sec_per_gpu": 43.85,
520
+ "tokens/trainable": 1200877
521
+ },
522
+ {
523
+ "epoch": 13.07843137254902,
524
+ "grad_norm": 0.7880769371986389,
525
+ "learning_rate": 1.736434530278362e-05,
526
+ "loss": 3.6283,
527
+ "memory/device_reserved (GiB)": 31.82,
528
+ "memory/max_active (GiB)": 30.03,
529
+ "memory/max_allocated (GiB)": 30.03,
530
+ "ppl": 37.64876,
531
+ "step": 170,
532
+ "tokens/total": 2732032,
533
+ "tokens/train_per_sec_per_gpu": 47.57,
534
+ "tokens/trainable": 1235550
535
+ },
536
+ {
537
+ "epoch": 13.470588235294118,
538
+ "grad_norm": 0.7571734189987183,
539
+ "learning_rate": 1.7193398003386514e-05,
540
+ "loss": 3.5842,
541
+ "memory/device_reserved (GiB)": 31.82,
542
+ "memory/max_active (GiB)": 30.03,
543
+ "memory/max_allocated (GiB)": 30.03,
544
+ "ppl": 36.02453,
545
+ "step": 175,
546
+ "tokens/total": 2813952,
547
+ "tokens/train_per_sec_per_gpu": 44.96,
548
+ "tokens/trainable": 1272589
549
+ },
550
+ {
551
+ "epoch": 13.862745098039216,
552
+ "grad_norm": 0.9505294561386108,
553
+ "learning_rate": 1.7017979028839918e-05,
554
+ "loss": 3.555,
555
+ "memory/device_reserved (GiB)": 31.82,
556
+ "memory/max_active (GiB)": 30.03,
557
+ "memory/max_allocated (GiB)": 30.03,
558
+ "ppl": 34.98782,
559
+ "step": 180,
560
+ "tokens/total": 2895872,
561
+ "tokens/train_per_sec_per_gpu": 43.68,
562
+ "tokens/trainable": 1309972
563
+ },
564
+ {
565
+ "epoch": 14.235294117647058,
566
+ "grad_norm": 1.3748526573181152,
567
+ "learning_rate": 1.68381974258932e-05,
568
+ "loss": 3.5266,
569
+ "memory/device_reserved (GiB)": 31.82,
570
+ "memory/max_active (GiB)": 30.03,
571
+ "memory/max_allocated (GiB)": 30.03,
572
+ "ppl": 34.00814,
573
+ "step": 185,
574
+ "tokens/total": 2973696,
575
+ "tokens/train_per_sec_per_gpu": 42.81,
576
+ "tokens/trainable": 1344786
577
+ },
578
+ {
579
+ "epoch": 14.627450980392156,
580
+ "grad_norm": 0.7306499481201172,
581
+ "learning_rate": 1.6654164953262614e-05,
582
+ "loss": 3.4729,
583
+ "memory/device_reserved (GiB)": 31.82,
584
+ "memory/max_active (GiB)": 30.03,
585
+ "memory/max_allocated (GiB)": 30.03,
586
+ "ppl": 32.23007,
587
+ "step": 190,
588
+ "tokens/total": 3055616,
589
+ "tokens/train_per_sec_per_gpu": 43.64,
590
+ "tokens/trainable": 1382206
591
+ },
592
+ {
593
+ "epoch": 15.0,
594
+ "grad_norm": 1.0910062789916992,
595
+ "learning_rate": 1.6465996012157996e-05,
596
+ "loss": 3.3893,
597
+ "memory/device_reserved (GiB)": 31.82,
598
+ "memory/max_active (GiB)": 30.03,
599
+ "memory/max_allocated (GiB)": 30.03,
600
+ "ppl": 29.64519,
601
+ "step": 195,
602
+ "tokens/total": 3133440,
603
+ "tokens/train_per_sec_per_gpu": 41.36,
604
+ "tokens/trainable": 1417035
605
+ },
606
+ {
607
+ "epoch": 15.392156862745098,
608
+ "grad_norm": 0.8951497673988342,
609
+ "learning_rate": 1.6273807575166927e-05,
610
+ "loss": 3.4152,
611
+ "memory/device_reserved (GiB)": 31.82,
612
+ "memory/max_active (GiB)": 30.03,
613
+ "memory/max_allocated (GiB)": 30.03,
614
+ "ppl": 30.42303,
615
+ "step": 200,
616
+ "tokens/total": 3215360,
617
+ "tokens/train_per_sec_per_gpu": 45.47,
618
+ "tokens/trainable": 1453690
619
+ },
620
+ {
621
+ "epoch": 15.392156862745098,
622
+ "eval_loss": 1.934309720993042,
623
+ "eval_ppl": 6.91927,
624
+ "eval_runtime": 52.4116,
625
+ "eval_samples_per_second": 3.816,
626
+ "eval_steps_per_second": 1.908,
627
+ "memory/device_reserved (GiB)": 31.82,
628
+ "memory/max_active (GiB)": 14.82,
629
+ "memory/max_allocated (GiB)": 14.82,
630
+ "step": 200
631
+ },
632
+ {
633
+ "epoch": 15.784313725490197,
634
+ "grad_norm": 0.8389677405357361,
635
+ "learning_rate": 1.6077719113540303e-05,
636
+ "loss": 3.3746,
637
+ "memory/device_reserved (GiB)": 31.82,
638
+ "memory/max_active (GiB)": 30.03,
639
+ "memory/max_allocated (GiB)": 30.03,
640
+ "ppl": 29.2126,
641
+ "step": 205,
642
+ "tokens/total": 3297280,
643
+ "tokens/train_per_sec_per_gpu": 46.15,
644
+ "tokens/trainable": 1491448
645
+ },
646
+ {
647
+ "epoch": 16.15686274509804,
648
+ "grad_norm": 0.9337797164916992,
649
+ "learning_rate": 1.5877852522924733e-05,
650
+ "loss": 3.3149,
651
+ "memory/device_reserved (GiB)": 31.82,
652
+ "memory/max_active (GiB)": 30.03,
653
+ "memory/max_allocated (GiB)": 30.03,
654
+ "ppl": 27.51964,
655
+ "step": 210,
656
+ "tokens/total": 3375104,
657
+ "tokens/train_per_sec_per_gpu": 45.59,
658
+ "tokens/trainable": 1526395
659
+ },
660
+ {
661
+ "epoch": 16.54901960784314,
662
+ "grad_norm": 0.8584134578704834,
663
+ "learning_rate": 1.567433204758782e-05,
664
+ "loss": 3.3092,
665
+ "memory/device_reserved (GiB)": 31.82,
666
+ "memory/max_active (GiB)": 30.03,
667
+ "memory/max_allocated (GiB)": 30.03,
668
+ "ppl": 27.36323,
669
+ "step": 215,
670
+ "tokens/total": 3457024,
671
+ "tokens/train_per_sec_per_gpu": 45.82,
672
+ "tokens/trainable": 1563348
673
+ },
674
+ {
675
+ "epoch": 16.941176470588236,
676
+ "grad_norm": 0.8377186059951782,
677
+ "learning_rate": 1.5467284203183437e-05,
678
+ "loss": 3.2949,
679
+ "memory/device_reserved (GiB)": 31.82,
680
+ "memory/max_active (GiB)": 30.03,
681
+ "memory/max_allocated (GiB)": 30.03,
682
+ "ppl": 26.97472,
683
+ "step": 220,
684
+ "tokens/total": 3538944,
685
+ "tokens/train_per_sec_per_gpu": 44.83,
686
+ "tokens/trainable": 1600934
687
+ },
688
+ {
689
+ "epoch": 17.313725490196077,
690
+ "grad_norm": 0.814207911491394,
691
+ "learning_rate": 1.5256837698105047e-05,
692
+ "loss": 3.2277,
693
+ "memory/device_reserved (GiB)": 31.82,
694
+ "memory/max_active (GiB)": 30.03,
695
+ "memory/max_allocated (GiB)": 30.03,
696
+ "ppl": 25.22158,
697
+ "step": 225,
698
+ "tokens/total": 3616768,
699
+ "tokens/train_per_sec_per_gpu": 46.04,
700
+ "tokens/trainable": 1636161
701
+ },
702
+ {
703
+ "epoch": 17.705882352941178,
704
+ "grad_norm": 0.8848176598548889,
705
+ "learning_rate": 1.5043123353475944e-05,
706
+ "loss": 3.2177,
707
+ "memory/device_reserved (GiB)": 31.82,
708
+ "memory/max_active (GiB)": 30.03,
709
+ "memory/max_allocated (GiB)": 30.03,
710
+ "ppl": 24.97062,
711
+ "step": 230,
712
+ "tokens/total": 3698688,
713
+ "tokens/train_per_sec_per_gpu": 45.18,
714
+ "tokens/trainable": 1673301
715
+ },
716
+ {
717
+ "epoch": 18.07843137254902,
718
+ "grad_norm": 0.847163736820221,
719
+ "learning_rate": 1.482627402182611e-05,
720
+ "loss": 3.2007,
721
+ "memory/device_reserved (GiB)": 31.82,
722
+ "memory/max_active (GiB)": 30.03,
723
+ "memory/max_allocated (GiB)": 30.03,
724
+ "ppl": 24.54971,
725
+ "step": 235,
726
+ "tokens/total": 3776512,
727
+ "tokens/train_per_sec_per_gpu": 42.62,
728
+ "tokens/trainable": 1707898
729
+ },
730
+ {
731
+ "epoch": 18.470588235294116,
732
+ "grad_norm": 0.8863293528556824,
733
+ "learning_rate": 1.4606424504506325e-05,
734
+ "loss": 3.1473,
735
+ "memory/device_reserved (GiB)": 31.82,
736
+ "memory/max_active (GiB)": 30.03,
737
+ "memory/max_allocated (GiB)": 30.03,
738
+ "ppl": 23.27314,
739
+ "step": 240,
740
+ "tokens/total": 3858432,
741
+ "tokens/train_per_sec_per_gpu": 46.4,
742
+ "tokens/trainable": 1745243
743
+ },
744
+ {
745
+ "epoch": 18.862745098039216,
746
+ "grad_norm": 0.8567431569099426,
747
+ "learning_rate": 1.4383711467890776e-05,
748
+ "loss": 3.1299,
749
+ "memory/device_reserved (GiB)": 31.82,
750
+ "memory/max_active (GiB)": 30.03,
751
+ "memory/max_allocated (GiB)": 30.03,
752
+ "ppl": 22.87169,
753
+ "step": 245,
754
+ "tokens/total": 3940352,
755
+ "tokens/train_per_sec_per_gpu": 46.63,
756
+ "tokens/trainable": 1782188
757
+ },
758
+ {
759
+ "epoch": 19.235294117647058,
760
+ "grad_norm": 0.8873140215873718,
761
+ "learning_rate": 1.415827335842033e-05,
762
+ "loss": 3.1617,
763
+ "memory/device_reserved (GiB)": 31.82,
764
+ "memory/max_active (GiB)": 30.03,
765
+ "memory/max_allocated (GiB)": 30.03,
766
+ "ppl": 23.6107,
767
+ "step": 250,
768
+ "tokens/total": 4018176,
769
+ "tokens/train_per_sec_per_gpu": 46.02,
770
+ "tokens/trainable": 1817232
771
+ },
772
+ {
773
+ "epoch": 19.235294117647058,
774
+ "eval_loss": 1.8731095790863037,
775
+ "eval_ppl": 6.5085,
776
+ "eval_runtime": 52.2712,
777
+ "eval_samples_per_second": 3.826,
778
+ "eval_steps_per_second": 1.913,
779
+ "memory/device_reserved (GiB)": 31.82,
780
+ "memory/max_active (GiB)": 14.82,
781
+ "memory/max_allocated (GiB)": 14.82,
782
+ "step": 250
783
+ },
784
+ {
785
+ "epoch": 19.627450980392158,
786
+ "grad_norm": 0.8352880477905273,
787
+ "learning_rate": 1.3930250316539237e-05,
788
+ "loss": 3.0786,
789
+ "memory/device_reserved (GiB)": 31.82,
790
+ "memory/max_active (GiB)": 30.03,
791
+ "memory/max_allocated (GiB)": 30.03,
792
+ "ppl": 21.72796,
793
+ "step": 255,
794
+ "tokens/total": 4100096,
795
+ "tokens/train_per_sec_per_gpu": 45.37,
796
+ "tokens/trainable": 1854866
797
+ },
798
+ {
799
+ "epoch": 20.0,
800
+ "grad_norm": 1.1509190797805786,
801
+ "learning_rate": 1.3699784089578791e-05,
802
+ "loss": 3.0425,
803
+ "memory/device_reserved (GiB)": 31.82,
804
+ "memory/max_active (GiB)": 30.03,
805
+ "memory/max_allocated (GiB)": 30.03,
806
+ "ppl": 20.95757,
807
+ "step": 260,
808
+ "tokens/total": 4177920,
809
+ "tokens/train_per_sec_per_gpu": 41.77,
810
+ "tokens/trainable": 1889380
811
+ },
812
+ {
813
+ "epoch": 20.392156862745097,
814
+ "grad_norm": 0.8712507486343384,
815
+ "learning_rate": 1.3467017943642074e-05,
816
+ "loss": 3.0706,
817
+ "memory/device_reserved (GiB)": 31.82,
818
+ "memory/max_active (GiB)": 30.03,
819
+ "memory/max_allocated (GiB)": 30.03,
820
+ "ppl": 21.55483,
821
+ "step": 265,
822
+ "tokens/total": 4259840,
823
+ "tokens/train_per_sec_per_gpu": 46.68,
824
+ "tokens/trainable": 1926452
825
+ },
826
+ {
827
+ "epoch": 20.784313725490197,
828
+ "grad_norm": 0.9600774645805359,
829
+ "learning_rate": 1.3232096574544602e-05,
830
+ "loss": 3.0228,
831
+ "memory/device_reserved (GiB)": 31.82,
832
+ "memory/max_active (GiB)": 30.03,
833
+ "memory/max_allocated (GiB)": 30.03,
834
+ "ppl": 20.54875,
835
+ "step": 270,
836
+ "tokens/total": 4341760,
837
+ "tokens/train_per_sec_per_gpu": 46.57,
838
+ "tokens/trainable": 1963989
839
+ },
840
+ {
841
+ "epoch": 21.15686274509804,
842
+ "grad_norm": 0.8765286803245544,
843
+ "learning_rate": 1.2995166017866194e-05,
844
+ "loss": 2.9804,
845
+ "memory/device_reserved (GiB)": 31.82,
846
+ "memory/max_active (GiB)": 30.03,
847
+ "memory/max_allocated (GiB)": 30.03,
848
+ "ppl": 19.69569,
849
+ "step": 275,
850
+ "tokens/total": 4419584,
851
+ "tokens/train_per_sec_per_gpu": 46.09,
852
+ "tokens/trainable": 1998647
853
+ },
854
+ {
855
+ "epoch": 21.54901960784314,
856
+ "grad_norm": 0.8645039200782776,
857
+ "learning_rate": 1.2756373558169992e-05,
858
+ "loss": 2.9896,
859
+ "memory/device_reserved (GiB)": 31.82,
860
+ "memory/max_active (GiB)": 30.03,
861
+ "memory/max_allocated (GiB)": 30.03,
862
+ "ppl": 19.87773,
863
+ "step": 280,
864
+ "tokens/total": 4501504,
865
+ "tokens/train_per_sec_per_gpu": 48.25,
866
+ "tokens/trainable": 2036645
867
+ },
868
+ {
869
+ "epoch": 21.941176470588236,
870
+ "grad_norm": 1.0529413223266602,
871
+ "learning_rate": 1.2515867637445088e-05,
872
+ "loss": 3.0176,
873
+ "memory/device_reserved (GiB)": 31.82,
874
+ "memory/max_active (GiB)": 30.03,
875
+ "memory/max_allocated (GiB)": 30.03,
876
+ "ppl": 20.44217,
877
+ "step": 285,
878
+ "tokens/total": 4583424,
879
+ "tokens/train_per_sec_per_gpu": 46.74,
880
+ "tokens/trainable": 2073360
881
+ },
882
+ {
883
+ "epoch": 22.313725490196077,
884
+ "grad_norm": 0.871498167514801,
885
+ "learning_rate": 1.2273797762829615e-05,
886
+ "loss": 2.9589,
887
+ "memory/device_reserved (GiB)": 31.82,
888
+ "memory/max_active (GiB)": 30.03,
889
+ "memory/max_allocated (GiB)": 30.03,
890
+ "ppl": 19.27676,
891
+ "step": 290,
892
+ "tokens/total": 4661248,
893
+ "tokens/train_per_sec_per_gpu": 46.09,
894
+ "tokens/trainable": 2108062
895
+ },
896
+ {
897
+ "epoch": 22.705882352941178,
898
+ "grad_norm": 0.8550048470497131,
899
+ "learning_rate": 1.2030314413671763e-05,
900
+ "loss": 2.9525,
901
+ "memory/device_reserved (GiB)": 31.82,
902
+ "memory/max_active (GiB)": 30.03,
903
+ "memory/max_allocated (GiB)": 30.03,
904
+ "ppl": 19.15378,
905
+ "step": 295,
906
+ "tokens/total": 4743168,
907
+ "tokens/train_per_sec_per_gpu": 50.48,
908
+ "tokens/trainable": 2145259
909
+ },
910
+ {
911
+ "epoch": 23.07843137254902,
912
+ "grad_norm": 0.8834354877471924,
913
+ "learning_rate": 1.1785568947986368e-05,
914
+ "loss": 2.9075,
915
+ "memory/device_reserved (GiB)": 31.82,
916
+ "memory/max_active (GiB)": 30.03,
917
+ "memory/max_allocated (GiB)": 30.03,
918
+ "ppl": 18.31096,
919
+ "step": 300,
920
+ "tokens/total": 4820992,
921
+ "tokens/train_per_sec_per_gpu": 45.14,
922
+ "tokens/trainable": 2180234
923
+ },
924
+ {
925
+ "epoch": 23.07843137254902,
926
+ "eval_loss": 1.8245855569839478,
927
+ "eval_ppl": 6.20022,
928
+ "eval_runtime": 52.3354,
929
+ "eval_samples_per_second": 3.822,
930
+ "eval_steps_per_second": 1.911,
931
+ "memory/device_reserved (GiB)": 31.82,
932
+ "memory/max_active (GiB)": 14.82,
933
+ "memory/max_allocated (GiB)": 14.82,
934
+ "step": 300
935
+ },
936
+ {
937
+ "epoch": 23.470588235294116,
938
+ "grad_norm": 0.9048839211463928,
939
+ "learning_rate": 1.1539713508365336e-05,
940
+ "loss": 2.9687,
941
+ "memory/device_reserved (GiB)": 31.82,
942
+ "memory/max_active (GiB)": 30.03,
943
+ "memory/max_allocated (GiB)": 30.03,
944
+ "ppl": 19.4666,
945
+ "step": 305,
946
+ "tokens/total": 4902912,
947
+ "tokens/train_per_sec_per_gpu": 40.14,
948
+ "tokens/trainable": 2217430
949
+ },
950
+ {
951
+ "epoch": 23.862745098039216,
952
+ "grad_norm": 0.8544889092445374,
953
+ "learning_rate": 1.1292900927400334e-05,
954
+ "loss": 2.8908,
955
+ "memory/device_reserved (GiB)": 31.82,
956
+ "memory/max_active (GiB)": 30.03,
957
+ "memory/max_allocated (GiB)": 30.03,
958
+ "ppl": 18.00771,
959
+ "step": 310,
960
+ "tokens/total": 4984832,
961
+ "tokens/train_per_sec_per_gpu": 44.78,
962
+ "tokens/trainable": 2254924
963
+ },
964
+ {
965
+ "epoch": 24.235294117647058,
966
+ "grad_norm": 0.9118690490722656,
967
+ "learning_rate": 1.1045284632676535e-05,
968
+ "loss": 2.8986,
969
+ "memory/device_reserved (GiB)": 31.82,
970
+ "memory/max_active (GiB)": 30.03,
971
+ "memory/max_allocated (GiB)": 30.03,
972
+ "ppl": 18.14872,
973
+ "step": 315,
974
+ "tokens/total": 5062656,
975
+ "tokens/train_per_sec_per_gpu": 48.84,
976
+ "tokens/trainable": 2289736
977
+ },
978
+ {
979
+ "epoch": 24.627450980392158,
980
+ "grad_norm": 0.9004949927330017,
981
+ "learning_rate": 1.0797018551396527e-05,
982
+ "loss": 2.8844,
983
+ "memory/device_reserved (GiB)": 31.82,
984
+ "memory/max_active (GiB)": 30.03,
985
+ "memory/max_allocated (GiB)": 30.03,
986
+ "ppl": 17.89283,
987
+ "step": 320,
988
+ "tokens/total": 5144576,
989
+ "tokens/train_per_sec_per_gpu": 49.81,
990
+ "tokens/trainable": 2327082
991
+ },
992
+ {
993
+ "epoch": 25.0,
994
+ "grad_norm": 1.1789140701293945,
995
+ "learning_rate": 1.0548257014693602e-05,
996
+ "loss": 2.8413,
997
+ "memory/device_reserved (GiB)": 31.82,
998
+ "memory/max_active (GiB)": 30.03,
999
+ "memory/max_allocated (GiB)": 30.03,
1000
+ "ppl": 17.13803,
1001
+ "step": 325,
1002
+ "tokens/total": 5222400,
1003
+ "tokens/train_per_sec_per_gpu": 44.28,
1004
+ "tokens/trainable": 2361725
1005
+ },
1006
+ {
1007
+ "epoch": 25.392156862745097,
1008
+ "grad_norm": 0.9994716048240662,
1009
+ "learning_rate": 1.0299154661693987e-05,
1010
+ "loss": 2.8495,
1011
+ "memory/device_reserved (GiB)": 31.82,
1012
+ "memory/max_active (GiB)": 30.03,
1013
+ "memory/max_allocated (GiB)": 30.03,
1014
+ "ppl": 17.27914,
1015
+ "step": 330,
1016
+ "tokens/total": 5304320,
1017
+ "tokens/train_per_sec_per_gpu": 44.32,
1018
+ "tokens/trainable": 2398889
1019
+ },
1020
+ {
1021
+ "epoch": 25.784313725490197,
1022
+ "grad_norm": 0.8554360866546631,
1023
+ "learning_rate": 1.0049866343387582e-05,
1024
+ "loss": 2.8331,
1025
+ "memory/device_reserved (GiB)": 31.82,
1026
+ "memory/max_active (GiB)": 30.03,
1027
+ "memory/max_allocated (GiB)": 30.03,
1028
+ "ppl": 16.99807,
1029
+ "step": 335,
1030
+ "tokens/total": 5386240,
1031
+ "tokens/train_per_sec_per_gpu": 49.6,
1032
+ "tokens/trainable": 2436175
1033
+ },
1034
+ {
1035
+ "epoch": 26.15686274509804,
1036
+ "grad_norm": 0.9382306337356567,
1037
+ "learning_rate": 9.800547026367022e-06,
1038
+ "loss": 2.8385,
1039
+ "memory/device_reserved (GiB)": 31.82,
1040
+ "memory/max_active (GiB)": 30.03,
1041
+ "memory/max_allocated (GiB)": 30.03,
1042
+ "ppl": 17.09011,
1043
+ "step": 340,
1044
+ "tokens/total": 5464064,
1045
+ "tokens/train_per_sec_per_gpu": 45.89,
1046
+ "tokens/trainable": 2471548
1047
+ },
1048
+ {
1049
+ "epoch": 26.54901960784314,
1050
+ "grad_norm": 1.0038572549819946,
1051
+ "learning_rate": 9.551351696494854e-06,
1052
+ "loss": 2.8006,
1053
+ "memory/device_reserved (GiB)": 31.82,
1054
+ "memory/max_active (GiB)": 30.03,
1055
+ "memory/max_allocated (GiB)": 30.03,
1056
+ "ppl": 16.45452,
1057
+ "step": 345,
1058
+ "tokens/total": 5545984,
1059
+ "tokens/train_per_sec_per_gpu": 44.95,
1060
+ "tokens/trainable": 2508740
1061
+ },
1062
+ {
1063
+ "epoch": 26.941176470588236,
1064
+ "grad_norm": 0.9205517768859863,
1065
+ "learning_rate": 9.302435262558748e-06,
1066
+ "loss": 2.8267,
1067
+ "memory/device_reserved (GiB)": 31.82,
1068
+ "memory/max_active (GiB)": 30.03,
1069
+ "memory/max_allocated (GiB)": 30.03,
1070
+ "ppl": 16.88963,
1071
+ "step": 350,
1072
+ "tokens/total": 5627904,
1073
+ "tokens/train_per_sec_per_gpu": 45.3,
1074
+ "tokens/trainable": 2545798
1075
+ },
1076
+ {
1077
+ "epoch": 26.941176470588236,
1078
+ "eval_loss": 1.7944940328598022,
1079
+ "eval_ppl": 6.01643,
1080
+ "eval_runtime": 52.4222,
1081
+ "eval_samples_per_second": 3.815,
1082
+ "eval_steps_per_second": 1.908,
1083
+ "memory/device_reserved (GiB)": 31.82,
1084
+ "memory/max_active (GiB)": 14.82,
1085
+ "memory/max_allocated (GiB)": 14.82,
1086
+ "step": 350
1087
+ },
1088
+ {
1089
+ "epoch": 27.313725490196077,
1090
+ "grad_norm": 0.8410999774932861,
1091
+ "learning_rate": 9.05395245997463e-06,
1092
+ "loss": 2.8095,
1093
+ "memory/device_reserved (GiB)": 31.82,
1094
+ "memory/max_active (GiB)": 30.03,
1095
+ "memory/max_allocated (GiB)": 30.03,
1096
+ "ppl": 16.60162,
1097
+ "step": 355,
1098
+ "tokens/total": 5705728,
1099
+ "tokens/train_per_sec_per_gpu": 42.61,
1100
+ "tokens/trainable": 2580317
1101
+ },
1102
+ {
1103
+ "epoch": 27.705882352941178,
1104
+ "grad_norm": 0.9195293188095093,
1105
+ "learning_rate": 8.806057754597559e-06,
1106
+ "loss": 2.8012,
1107
+ "memory/device_reserved (GiB)": 31.82,
1108
+ "memory/max_active (GiB)": 30.03,
1109
+ "memory/max_allocated (GiB)": 30.03,
1110
+ "ppl": 16.46439,
1111
+ "step": 360,
1112
+ "tokens/total": 5787648,
1113
+ "tokens/train_per_sec_per_gpu": 47.59,
1114
+ "tokens/trainable": 2618023
1115
+ },
1116
+ {
1117
+ "epoch": 28.07843137254902,
1118
+ "grad_norm": 1.0124417543411255,
1119
+ "learning_rate": 8.558905246700202e-06,
1120
+ "loss": 2.8255,
1121
+ "memory/device_reserved (GiB)": 31.82,
1122
+ "memory/max_active (GiB)": 30.03,
1123
+ "memory/max_allocated (GiB)": 30.03,
1124
+ "ppl": 16.86938,
1125
+ "step": 365,
1126
+ "tokens/total": 5865472,
1127
+ "tokens/train_per_sec_per_gpu": 46.39,
1128
+ "tokens/trainable": 2652689
1129
+ },
1130
+ {
1131
+ "epoch": 28.470588235294116,
1132
+ "grad_norm": 1.0995986461639404,
1133
+ "learning_rate": 8.312648575178552e-06,
1134
+ "loss": 2.79,
1135
+ "memory/device_reserved (GiB)": 31.82,
1136
+ "memory/max_active (GiB)": 30.03,
1137
+ "memory/max_allocated (GiB)": 30.03,
1138
+ "ppl": 16.28102,
1139
+ "step": 370,
1140
+ "tokens/total": 5947392,
1141
+ "tokens/train_per_sec_per_gpu": 46.65,
1142
+ "tokens/trainable": 2690200
1143
+ },
1144
+ {
1145
+ "epoch": 28.862745098039216,
1146
+ "grad_norm": 0.9146213531494141,
1147
+ "learning_rate": 8.06744082204447e-06,
1148
+ "loss": 2.7736,
1149
+ "memory/device_reserved (GiB)": 31.82,
1150
+ "memory/max_active (GiB)": 30.03,
1151
+ "memory/max_allocated (GiB)": 30.03,
1152
+ "ppl": 16.01619,
1153
+ "step": 375,
1154
+ "tokens/total": 6029312,
1155
+ "tokens/train_per_sec_per_gpu": 44.2,
1156
+ "tokens/trainable": 2727130
1157
+ },
1158
+ {
1159
+ "epoch": 29.235294117647058,
1160
+ "grad_norm": 1.1337180137634277,
1161
+ "learning_rate": 7.823434417264378e-06,
1162
+ "loss": 2.7499,
1163
+ "memory/device_reserved (GiB)": 31.82,
1164
+ "memory/max_active (GiB)": 30.03,
1165
+ "memory/max_allocated (GiB)": 30.03,
1166
+ "ppl": 15.64107,
1167
+ "step": 380,
1168
+ "tokens/total": 6107136,
1169
+ "tokens/train_per_sec_per_gpu": 44.25,
1170
+ "tokens/trainable": 2761625
1171
+ },
1172
+ {
1173
+ "epoch": 29.627450980392158,
1174
+ "grad_norm": 0.8825079202651978,
1175
+ "learning_rate": 7.580781044003324e-06,
1176
+ "loss": 2.7665,
1177
+ "memory/device_reserved (GiB)": 31.82,
1178
+ "memory/max_active (GiB)": 30.03,
1179
+ "memory/max_allocated (GiB)": 30.03,
1180
+ "ppl": 15.90288,
1181
+ "step": 385,
1182
+ "tokens/total": 6189056,
1183
+ "tokens/train_per_sec_per_gpu": 44.98,
1184
+ "tokens/trainable": 2799048
1185
+ },
1186
+ {
1187
+ "epoch": 30.0,
1188
+ "grad_norm": 1.2486763000488281,
1189
+ "learning_rate": 7.33963154433325e-06,
1190
+ "loss": 2.7715,
1191
+ "memory/device_reserved (GiB)": 31.82,
1192
+ "memory/max_active (GiB)": 30.03,
1193
+ "memory/max_allocated (GiB)": 30.03,
1194
+ "ppl": 15.98259,
1195
+ "step": 390,
1196
+ "tokens/total": 6266880,
1197
+ "tokens/train_per_sec_per_gpu": 44.08,
1198
+ "tokens/trainable": 2834070
1199
+ },
1200
+ {
1201
+ "epoch": 30.392156862745097,
1202
+ "grad_norm": 1.1907079219818115,
1203
+ "learning_rate": 7.100135825464138e-06,
1204
+ "loss": 2.7558,
1205
+ "memory/device_reserved (GiB)": 31.82,
1206
+ "memory/max_active (GiB)": 30.03,
1207
+ "memory/max_allocated (GiB)": 30.03,
1208
+ "ppl": 15.73362,
1209
+ "step": 395,
1210
+ "tokens/total": 6348800,
1211
+ "tokens/train_per_sec_per_gpu": 48.43,
1212
+ "tokens/trainable": 2871536
1213
+ },
1214
+ {
1215
+ "epoch": 30.784313725490197,
1216
+ "grad_norm": 0.8881621360778809,
1217
+ "learning_rate": 6.862442766556297e-06,
1218
+ "loss": 2.7239,
1219
+ "memory/device_reserved (GiB)": 31.82,
1220
+ "memory/max_active (GiB)": 30.03,
1221
+ "memory/max_allocated (GiB)": 30.03,
1222
+ "ppl": 15.23964,
1223
+ "step": 400,
1224
+ "tokens/total": 6430720,
1225
+ "tokens/train_per_sec_per_gpu": 43.32,
1226
+ "tokens/trainable": 2909069
1227
+ },
1228
+ {
1229
+ "epoch": 30.784313725490197,
1230
+ "eval_loss": 1.7793785333633423,
1231
+ "eval_ppl": 5.92617,
1232
+ "eval_runtime": 52.3688,
1233
+ "eval_samples_per_second": 3.819,
1234
+ "eval_steps_per_second": 1.91,
1235
+ "memory/device_reserved (GiB)": 31.82,
1236
+ "memory/max_active (GiB)": 14.82,
1237
+ "memory/max_allocated (GiB)": 14.82,
1238
+ "step": 400
1239
+ }
1240
+ ],
1241
+ "logging_steps": 5,
1242
+ "max_steps": 650,
1243
+ "num_input_tokens_seen": 0,
1244
+ "num_train_epochs": 50,
1245
+ "save_steps": 100,
1246
+ "stateful_callbacks": {
1247
+ "TrainerControl": {
1248
+ "args": {
1249
+ "should_epoch_stop": false,
1250
+ "should_evaluate": false,
1251
+ "should_log": false,
1252
+ "should_save": true,
1253
+ "should_training_stop": false
1254
+ },
1255
+ "attributes": {}
1256
+ }
1257
+ },
1258
+ "total_flos": 1.2435718638927872e+17,
1259
+ "train_batch_size": 2,
1260
+ "trial_name": null,
1261
+ "trial_params": null
1262
+ }
checkpoint-400/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a94bb930d8dd65dcaaaff77c43d92a52c500cdd1a9193a036a0d8e7038b4e8d1
3
+ size 9425
checkpoint-400/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-400/zero_to_fp32.py ADDED
@@ -0,0 +1,760 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) Microsoft Corporation.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # DeepSpeed Team
7
+
8
+ # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
+ # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
+ # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
+ # application.
12
+ #
13
+ # example:
14
+ # python zero_to_fp32.py . output_dir/
15
+ # or
16
+ # python zero_to_fp32.py . output_dir/ --safe_serialization
17
+
18
+ import argparse
19
+ import torch
20
+ import glob
21
+ import math
22
+ import os
23
+ import re
24
+ import gc
25
+ import json
26
+ import numpy as np
27
+ from tqdm import tqdm
28
+ from collections import OrderedDict
29
+ from dataclasses import dataclass
30
+
31
+ # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
32
+ # DeepSpeed data structures it has to be available in the current python environment.
33
+ from deepspeed.utils import logger
34
+ from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
35
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
36
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
37
+
38
+
39
+ @dataclass
40
+ class zero_model_state:
41
+ buffers: dict()
42
+ param_shapes: dict()
43
+ shared_params: list
44
+ ds_version: int
45
+ frozen_param_shapes: dict()
46
+ frozen_param_fragments: dict()
47
+
48
+
49
+ debug = 0
50
+
51
+ # load to cpu
52
+ device = torch.device('cpu')
53
+
54
+
55
+ def atoi(text):
56
+ return int(text) if text.isdigit() else text
57
+
58
+
59
+ def natural_keys(text):
60
+ '''
61
+ alist.sort(key=natural_keys) sorts in human order
62
+ http://nedbatchelder.com/blog/200712/human_sorting.html
63
+ (See Toothy's implementation in the comments)
64
+ '''
65
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
66
+
67
+
68
+ def get_model_state_file(checkpoint_dir, zero_stage):
69
+ if not os.path.isdir(checkpoint_dir):
70
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
71
+
72
+ # there should be only one file
73
+ if zero_stage <= 2:
74
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
75
+ elif zero_stage == 3:
76
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
77
+
78
+ if not os.path.exists(file):
79
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
80
+
81
+ return file
82
+
83
+
84
+ def get_checkpoint_files(checkpoint_dir, glob_pattern):
85
+ # XXX: need to test that this simple glob rule works for multi-node setup too
86
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
87
+
88
+ if len(ckpt_files) == 0:
89
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
90
+
91
+ return ckpt_files
92
+
93
+
94
+ def get_optim_files(checkpoint_dir):
95
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
96
+
97
+
98
+ def get_model_state_files(checkpoint_dir):
99
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
100
+
101
+
102
+ def parse_model_states(files):
103
+ zero_model_states = []
104
+ for file in files:
105
+ state_dict = torch.load(file, map_location=device, weights_only=False)
106
+
107
+ if BUFFER_NAMES not in state_dict:
108
+ raise ValueError(f"{file} is not a model state checkpoint")
109
+ buffer_names = state_dict[BUFFER_NAMES]
110
+ if debug:
111
+ print("Found buffers:", buffer_names)
112
+
113
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
114
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
115
+ param_shapes = state_dict[PARAM_SHAPES]
116
+
117
+ # collect parameters that are included in param_shapes
118
+ param_names = []
119
+ for s in param_shapes:
120
+ for name in s.keys():
121
+ param_names.append(name)
122
+
123
+ # update with frozen parameters
124
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
125
+ if frozen_param_shapes is not None:
126
+ if debug:
127
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
128
+ param_names += list(frozen_param_shapes.keys())
129
+
130
+ # handle shared params
131
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
132
+
133
+ ds_version = state_dict.get(DS_VERSION, None)
134
+
135
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
136
+
137
+ z_model_state = zero_model_state(buffers=buffers,
138
+ param_shapes=param_shapes,
139
+ shared_params=shared_params,
140
+ ds_version=ds_version,
141
+ frozen_param_shapes=frozen_param_shapes,
142
+ frozen_param_fragments=frozen_param_fragments)
143
+ zero_model_states.append(z_model_state)
144
+
145
+ return zero_model_states
146
+
147
+
148
+ def parse_optim_states(files, ds_checkpoint_dir):
149
+ total_files = len(files)
150
+ state_dicts = []
151
+ for f in tqdm(files, desc='Loading checkpoint shards'):
152
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
153
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
154
+ # and also handle the case where it was already removed by another helper script
155
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
156
+ state_dicts.append(state_dict)
157
+
158
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
159
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
160
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
161
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
162
+
163
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
164
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
165
+ # use the max of the partition_count to get the dp world_size.
166
+
167
+ if type(world_size) is list:
168
+ world_size = max(world_size)
169
+
170
+ if world_size != total_files:
171
+ raise ValueError(
172
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
173
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
174
+ )
175
+
176
+ # the groups are named differently in each stage
177
+ if zero_stage <= 2:
178
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
179
+ elif zero_stage == 3:
180
+ fp32_groups_key = FP32_FLAT_GROUPS
181
+ else:
182
+ raise ValueError(f"unknown zero stage {zero_stage}")
183
+
184
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
185
+ return zero_stage, world_size, fp32_flat_groups
186
+
187
+
188
+ def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
189
+ """
190
+ Returns fp32 state_dict reconstructed from ds checkpoint
191
+
192
+ Args:
193
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
194
+
195
+ """
196
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
197
+
198
+ optim_files = get_optim_files(ds_checkpoint_dir)
199
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
200
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
201
+
202
+ model_files = get_model_state_files(ds_checkpoint_dir)
203
+
204
+ zero_model_states = parse_model_states(model_files)
205
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
206
+
207
+ if zero_stage <= 2:
208
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
209
+ exclude_frozen_parameters)
210
+ elif zero_stage == 3:
211
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
212
+ exclude_frozen_parameters)
213
+
214
+
215
+ def _zero2_merge_frozen_params(state_dict, zero_model_states):
216
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
217
+ return
218
+
219
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
220
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
221
+
222
+ if debug:
223
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
224
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
225
+
226
+ wanted_params = len(frozen_param_shapes)
227
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
228
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
229
+ print(f'Frozen params: Have {avail_numel} numels to process.')
230
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
231
+
232
+ total_params = 0
233
+ total_numel = 0
234
+ for name, shape in frozen_param_shapes.items():
235
+ total_params += 1
236
+ unpartitioned_numel = shape.numel()
237
+ total_numel += unpartitioned_numel
238
+
239
+ state_dict[name] = frozen_param_fragments[name]
240
+
241
+ if debug:
242
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
243
+
244
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
245
+
246
+
247
+ def _has_callable(obj, fn):
248
+ attr = getattr(obj, fn, None)
249
+ return callable(attr)
250
+
251
+
252
+ def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
253
+ param_shapes = zero_model_states[0].param_shapes
254
+
255
+ # Reconstruction protocol:
256
+ #
257
+ # XXX: document this
258
+
259
+ if debug:
260
+ for i in range(world_size):
261
+ for j in range(len(fp32_flat_groups[0])):
262
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
263
+
264
+ # XXX: memory usage doubles here (zero2)
265
+ num_param_groups = len(fp32_flat_groups[0])
266
+ merged_single_partition_of_fp32_groups = []
267
+ for i in range(num_param_groups):
268
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
269
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
270
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
271
+ avail_numel = sum(
272
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
273
+
274
+ if debug:
275
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
276
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
277
+ # not asserting if there is a mismatch due to possible padding
278
+ print(f"Have {avail_numel} numels to process.")
279
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
280
+
281
+ # params
282
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
283
+ # out-of-core computing solution
284
+ total_numel = 0
285
+ total_params = 0
286
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
287
+ offset = 0
288
+ avail_numel = full_single_fp32_vector.numel()
289
+ for name, shape in shapes.items():
290
+
291
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
292
+ total_numel += unpartitioned_numel
293
+ total_params += 1
294
+
295
+ if debug:
296
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
297
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
298
+ offset += unpartitioned_numel
299
+
300
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
301
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
302
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
303
+ # live optimizer object, so we are checking that the numbers are within the right range
304
+ align_to = 2 * world_size
305
+
306
+ def zero2_align(x):
307
+ return align_to * math.ceil(x / align_to)
308
+
309
+ if debug:
310
+ print(f"original offset={offset}, avail_numel={avail_numel}")
311
+
312
+ offset = zero2_align(offset)
313
+ avail_numel = zero2_align(avail_numel)
314
+
315
+ if debug:
316
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
317
+
318
+ # Sanity check
319
+ if offset != avail_numel:
320
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
321
+
322
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
323
+
324
+
325
+ def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
326
+ exclude_frozen_parameters):
327
+ state_dict = OrderedDict()
328
+
329
+ # buffers
330
+ buffers = zero_model_states[0].buffers
331
+ state_dict.update(buffers)
332
+ if debug:
333
+ print(f"added {len(buffers)} buffers")
334
+
335
+ if not exclude_frozen_parameters:
336
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
337
+
338
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
339
+
340
+ # recover shared parameters
341
+ for pair in zero_model_states[0].shared_params:
342
+ if pair[1] in state_dict:
343
+ state_dict[pair[0]] = state_dict[pair[1]]
344
+
345
+ return state_dict
346
+
347
+
348
+ def zero3_partitioned_param_info(unpartitioned_numel, world_size):
349
+ remainder = unpartitioned_numel % world_size
350
+ padding_numel = (world_size - remainder) if remainder else 0
351
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
352
+ return partitioned_numel, padding_numel
353
+
354
+
355
+ def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
356
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
357
+ return
358
+
359
+ if debug:
360
+ for i in range(world_size):
361
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
362
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
363
+
364
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
365
+ wanted_params = len(frozen_param_shapes)
366
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
367
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
368
+ print(f'Frozen params: Have {avail_numel} numels to process.')
369
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
370
+
371
+ total_params = 0
372
+ total_numel = 0
373
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
374
+ total_params += 1
375
+ unpartitioned_numel = shape.numel()
376
+ total_numel += unpartitioned_numel
377
+
378
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
379
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
380
+
381
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
382
+
383
+ if debug:
384
+ print(
385
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
386
+ )
387
+
388
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
389
+
390
+
391
+ class GatheredTensor:
392
+ """
393
+ A pseudo tensor that collects partitioned weights.
394
+ It is more memory efficient when there are multiple groups.
395
+ """
396
+
397
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
398
+ self.flat_groups = flat_groups
399
+ self.flat_groups_offset = flat_groups_offset
400
+ self.offset = offset
401
+ self.partitioned_numel = partitioned_numel
402
+ self.shape = shape
403
+ self.dtype = self.flat_groups[0][0].dtype
404
+
405
+ def contiguous(self):
406
+ """
407
+ Merge partitioned weights from flat_groups into a single tensor.
408
+ """
409
+ end_idx = self.offset + self.partitioned_numel
410
+ world_size = len(self.flat_groups)
411
+ pad_flat_param_chunks = []
412
+
413
+ for rank_i in range(world_size):
414
+ # for each rank, we need to collect weights from related group/groups
415
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
416
+ start_group_id = None
417
+ end_group_id = None
418
+ for group_id in range(len(self.flat_groups_offset)):
419
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
420
+ start_group_id = group_id
421
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
422
+ end_group_id = group_id
423
+ break
424
+ # collect weights from related group/groups
425
+ for group_id in range(start_group_id, end_group_id + 1):
426
+ flat_tensor = flat_groups_at_rank_i[group_id]
427
+ start_offset = self.offset - self.flat_groups_offset[group_id]
428
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
429
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
430
+
431
+ # collect weights from all ranks
432
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
433
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
434
+ return param
435
+
436
+
437
+ def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
438
+ param_shapes = zero_model_states[0].param_shapes
439
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
440
+
441
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
442
+ # param, re-consolidating each param, while dealing with padding if any
443
+
444
+ # merge list of dicts, preserving order
445
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
446
+
447
+ if debug:
448
+ for i in range(world_size):
449
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
450
+
451
+ wanted_params = len(param_shapes)
452
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
453
+ # not asserting if there is a mismatch due to possible padding
454
+ avail_numel = fp32_flat_groups[0].numel() * world_size
455
+ print(f"Trainable params: Have {avail_numel} numels to process.")
456
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
457
+
458
+ # params
459
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
460
+ # out-of-core computing solution
461
+ offset = 0
462
+ total_numel = 0
463
+ total_params = 0
464
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
465
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
466
+ unpartitioned_numel = shape.numel()
467
+ total_numel += unpartitioned_numel
468
+ total_params += 1
469
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
470
+
471
+ if debug:
472
+ print(
473
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
474
+ )
475
+
476
+ # memory efficient tensor
477
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
478
+ state_dict[name] = tensor
479
+ offset += partitioned_numel
480
+
481
+ offset *= world_size
482
+
483
+ # Sanity check
484
+ if offset != avail_numel:
485
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
486
+
487
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
488
+
489
+
490
+ def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
491
+ exclude_frozen_parameters):
492
+ state_dict = OrderedDict()
493
+
494
+ # buffers
495
+ buffers = zero_model_states[0].buffers
496
+ state_dict.update(buffers)
497
+ if debug:
498
+ print(f"added {len(buffers)} buffers")
499
+
500
+ if not exclude_frozen_parameters:
501
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
502
+
503
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
504
+
505
+ # recover shared parameters
506
+ for pair in zero_model_states[0].shared_params:
507
+ if pair[1] in state_dict:
508
+ state_dict[pair[0]] = state_dict[pair[1]]
509
+
510
+ return state_dict
511
+
512
+
513
+ def to_torch_tensor(state_dict, return_empty_tensor=False):
514
+ """
515
+ Convert state_dict of GatheredTensor to torch tensor
516
+ """
517
+ torch_state_dict = {}
518
+ converted_tensors = {}
519
+ for name, tensor in state_dict.items():
520
+ tensor_id = id(tensor)
521
+ if tensor_id in converted_tensors: # shared tensors
522
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
523
+ torch_state_dict[name] = shared_tensor
524
+ else:
525
+ converted_tensors[tensor_id] = name
526
+ if return_empty_tensor:
527
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
528
+ else:
529
+ torch_state_dict[name] = tensor.contiguous()
530
+ return torch_state_dict
531
+
532
+
533
+ def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
534
+ tag=None,
535
+ exclude_frozen_parameters=False,
536
+ lazy_mode=False):
537
+ """
538
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
539
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
540
+ via a model hub.
541
+
542
+ Args:
543
+ - ``checkpoint_dir``: path to the desired checkpoint folder
544
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
545
+ - ``exclude_frozen_parameters``: exclude frozen parameters
546
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
547
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
548
+
549
+ Returns:
550
+ - pytorch ``state_dict``
551
+
552
+ A typical usage might be ::
553
+
554
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
555
+ # do the training and checkpoint saving
556
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
557
+ model = model.cpu() # move to cpu
558
+ model.load_state_dict(state_dict)
559
+ # submit to model hub or save the model to share with others
560
+
561
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
562
+ application. i.e. you will need to re-initialize the deepspeed engine, since
563
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
564
+
565
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
566
+
567
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
568
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
569
+ the checkpoint. Or you can load state_dict in lazy mode ::
570
+
571
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
572
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
573
+ for name, lazy_tensor in state_dict.item():
574
+ tensor = lazy_tensor.contiguous() # to cpu
575
+ print(name, tensor)
576
+ # del tensor to release memory if it no longer in use
577
+ """
578
+ if tag is None:
579
+ latest_path = os.path.join(checkpoint_dir, 'latest')
580
+ if os.path.isfile(latest_path):
581
+ with open(latest_path, 'r') as fd:
582
+ tag = fd.read().strip()
583
+ else:
584
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
585
+
586
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
587
+
588
+ if not os.path.isdir(ds_checkpoint_dir):
589
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
590
+
591
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
592
+ if lazy_mode:
593
+ return state_dict
594
+ else:
595
+ return to_torch_tensor(state_dict)
596
+
597
+
598
+ def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
599
+ output_dir,
600
+ max_shard_size="5GB",
601
+ safe_serialization=False,
602
+ tag=None,
603
+ exclude_frozen_parameters=False):
604
+ """
605
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
606
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
607
+
608
+ Args:
609
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
610
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
611
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
612
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
613
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
614
+ - ``exclude_frozen_parameters``: exclude frozen parameters
615
+ """
616
+
617
+ # Dependency pre-check
618
+ if safe_serialization:
619
+ try:
620
+ from safetensors.torch import save_file
621
+ except ImportError:
622
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
623
+ raise
624
+ if max_shard_size is not None:
625
+ try:
626
+ from huggingface_hub import split_torch_state_dict_into_shards
627
+ except ImportError:
628
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
629
+ raise
630
+
631
+ # Convert zero checkpoint to state_dict
632
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
633
+ tag,
634
+ exclude_frozen_parameters,
635
+ lazy_mode=True)
636
+
637
+ # Shard the model if it is too big.
638
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
639
+ if max_shard_size is not None:
640
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
641
+ # an memory-efficient approach for sharding
642
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
643
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
644
+ filename_pattern=filename_pattern,
645
+ max_shard_size=max_shard_size)
646
+ else:
647
+ from collections import namedtuple
648
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
649
+ state_dict_split = StateDictSplit(is_sharded=False,
650
+ filename_to_tensors={weights_name: list(state_dict.keys())})
651
+
652
+ # Save the model by shard
653
+ os.makedirs(output_dir, exist_ok=True)
654
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
655
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
656
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
657
+ shard_state_dict = to_torch_tensor(shard_state_dict)
658
+ output_path = os.path.join(output_dir, shard_file)
659
+ if safe_serialization:
660
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
661
+ else:
662
+ torch.save(shard_state_dict, output_path)
663
+ # release the memory of current shard
664
+ for tensor_name in list(shard_state_dict.keys()):
665
+ del state_dict[tensor_name]
666
+ del shard_state_dict[tensor_name]
667
+ del shard_state_dict
668
+ gc.collect()
669
+
670
+ # Save index if sharded
671
+ if state_dict_split.is_sharded:
672
+ index = {
673
+ "metadata": state_dict_split.metadata,
674
+ "weight_map": state_dict_split.tensor_to_filename,
675
+ }
676
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
677
+ save_index_file = os.path.join(output_dir, save_index_file)
678
+ with open(save_index_file, "w", encoding="utf-8") as f:
679
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
680
+ f.write(content)
681
+
682
+
683
+ def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
684
+ """
685
+ 1. Put the provided model to cpu
686
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
687
+ 3. Load it into the provided model
688
+
689
+ Args:
690
+ - ``model``: the model object to update
691
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
692
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
693
+
694
+ Returns:
695
+ - ``model`: modified model
696
+
697
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
698
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
699
+ conveniently placed for you in the checkpoint folder.
700
+
701
+ A typical usage might be ::
702
+
703
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
704
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
705
+ # submit to model hub or save the model to share with others
706
+
707
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
708
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
709
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
710
+
711
+ """
712
+ logger.info("Extracting fp32 weights")
713
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
714
+
715
+ logger.info("Overwriting model with fp32 weights")
716
+ model = model.cpu()
717
+ model.load_state_dict(state_dict, strict=False)
718
+
719
+ return model
720
+
721
+
722
+ if __name__ == "__main__":
723
+ parser = argparse.ArgumentParser()
724
+ parser.add_argument("checkpoint_dir",
725
+ type=str,
726
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
727
+ parser.add_argument("output_dir",
728
+ type=str,
729
+ help="directory to the pytorch fp32 state_dict output files"
730
+ "(e.g. path/checkpoint-12-output/)")
731
+ parser.add_argument(
732
+ "--max_shard_size",
733
+ type=str,
734
+ default="5GB",
735
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
736
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
737
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
738
+ "without CPU OOM issues.")
739
+ parser.add_argument(
740
+ "--safe_serialization",
741
+ default=False,
742
+ action='store_true',
743
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
744
+ parser.add_argument("-t",
745
+ "--tag",
746
+ type=str,
747
+ default=None,
748
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
749
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
750
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
751
+ args = parser.parse_args()
752
+
753
+ debug = args.debug
754
+
755
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
756
+ args.output_dir,
757
+ max_shard_size=args.max_shard_size,
758
+ safe_serialization=args.safe_serialization,
759
+ tag=args.tag,
760
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
checkpoint-500/README.md ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: microsoft/Phi-4-mini-instruct
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - axolotl
7
+ - base_model:adapter:microsoft/Phi-4-mini-instruct
8
+ - lora
9
+ - transformers
10
+ ---
11
+
12
+ # Model Card for Model ID
13
+
14
+ <!-- Provide a quick summary of what the model is/does. -->
15
+
16
+
17
+
18
+ ## Model Details
19
+
20
+ ### Model Description
21
+
22
+ <!-- Provide a longer summary of what this model is. -->
23
+
24
+
25
+
26
+ - **Developed by:** [More Information Needed]
27
+ - **Funded by [optional]:** [More Information Needed]
28
+ - **Shared by [optional]:** [More Information Needed]
29
+ - **Model type:** [More Information Needed]
30
+ - **Language(s) (NLP):** [More Information Needed]
31
+ - **License:** [More Information Needed]
32
+ - **Finetuned from model [optional]:** [More Information Needed]
33
+
34
+ ### Model Sources [optional]
35
+
36
+ <!-- Provide the basic links for the model. -->
37
+
38
+ - **Repository:** [More Information Needed]
39
+ - **Paper [optional]:** [More Information Needed]
40
+ - **Demo [optional]:** [More Information Needed]
41
+
42
+ ## Uses
43
+
44
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
45
+
46
+ ### Direct Use
47
+
48
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
49
+
50
+ [More Information Needed]
51
+
52
+ ### Downstream Use [optional]
53
+
54
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
55
+
56
+ [More Information Needed]
57
+
58
+ ### Out-of-Scope Use
59
+
60
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
61
+
62
+ [More Information Needed]
63
+
64
+ ## Bias, Risks, and Limitations
65
+
66
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
67
+
68
+ [More Information Needed]
69
+
70
+ ### Recommendations
71
+
72
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
73
+
74
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
75
+
76
+ ## How to Get Started with the Model
77
+
78
+ Use the code below to get started with the model.
79
+
80
+ [More Information Needed]
81
+
82
+ ## Training Details
83
+
84
+ ### Training Data
85
+
86
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
87
+
88
+ [More Information Needed]
89
+
90
+ ### Training Procedure
91
+
92
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
93
+
94
+ #### Preprocessing [optional]
95
+
96
+ [More Information Needed]
97
+
98
+
99
+ #### Training Hyperparameters
100
+
101
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
102
+
103
+ #### Speeds, Sizes, Times [optional]
104
+
105
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
106
+
107
+ [More Information Needed]
108
+
109
+ ## Evaluation
110
+
111
+ <!-- This section describes the evaluation protocols and provides the results. -->
112
+
113
+ ### Testing Data, Factors & Metrics
114
+
115
+ #### Testing Data
116
+
117
+ <!-- This should link to a Dataset Card if possible. -->
118
+
119
+ [More Information Needed]
120
+
121
+ #### Factors
122
+
123
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
124
+
125
+ [More Information Needed]
126
+
127
+ #### Metrics
128
+
129
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
130
+
131
+ [More Information Needed]
132
+
133
+ ### Results
134
+
135
+ [More Information Needed]
136
+
137
+ #### Summary
138
+
139
+
140
+
141
+ ## Model Examination [optional]
142
+
143
+ <!-- Relevant interpretability work for the model goes here -->
144
+
145
+ [More Information Needed]
146
+
147
+ ## Environmental Impact
148
+
149
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
150
+
151
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
152
+
153
+ - **Hardware Type:** [More Information Needed]
154
+ - **Hours used:** [More Information Needed]
155
+ - **Cloud Provider:** [More Information Needed]
156
+ - **Compute Region:** [More Information Needed]
157
+ - **Carbon Emitted:** [More Information Needed]
158
+
159
+ ## Technical Specifications [optional]
160
+
161
+ ### Model Architecture and Objective
162
+
163
+ [More Information Needed]
164
+
165
+ ### Compute Infrastructure
166
+
167
+ [More Information Needed]
168
+
169
+ #### Hardware
170
+
171
+ [More Information Needed]
172
+
173
+ #### Software
174
+
175
+ [More Information Needed]
176
+
177
+ ## Citation [optional]
178
+
179
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
180
+
181
+ **BibTeX:**
182
+
183
+ [More Information Needed]
184
+
185
+ **APA:**
186
+
187
+ [More Information Needed]
188
+
189
+ ## Glossary [optional]
190
+
191
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
192
+
193
+ [More Information Needed]
194
+
195
+ ## More Information [optional]
196
+
197
+ [More Information Needed]
198
+
199
+ ## Model Card Authors [optional]
200
+
201
+ [More Information Needed]
202
+
203
+ ## Model Card Contact
204
+
205
+ [More Information Needed]
206
+ ### Framework versions
207
+
208
+ - PEFT 0.18.1
checkpoint-500/adapter_config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "microsoft/Phi-4-mini-instruct",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": null,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 16,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.05,
22
+ "megatron_config": null,
23
+ "megatron_core": "megatron.core",
24
+ "modules_to_save": null,
25
+ "peft_type": "LORA",
26
+ "peft_version": "0.18.1",
27
+ "qalora_group_size": 16,
28
+ "r": 8,
29
+ "rank_pattern": {},
30
+ "revision": null,
31
+ "target_modules": [
32
+ "k_proj",
33
+ "q_proj",
34
+ "v_proj",
35
+ "o_proj"
36
+ ],
37
+ "target_parameters": [],
38
+ "task_type": "CAUSAL_LM",
39
+ "trainable_token_indices": null,
40
+ "use_dora": false,
41
+ "use_qalora": false,
42
+ "use_rslora": false
43
+ }
checkpoint-500/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68fdb7a6a9af7867a714d236ef533e8807110e0eee82ee8aff5bc74f22a1156a
3
+ size 3154216
checkpoint-500/added_tokens.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|/tool_call|>": 200026,
3
+ "<|/tool|>": 200024,
4
+ "<|assistant|>": 200019,
5
+ "<|end|>": 200020,
6
+ "<|system|>": 200022,
7
+ "<|tag|>": 200028,
8
+ "<|tool_call|>": 200025,
9
+ "<|tool_response|>": 200027,
10
+ "<|tool|>": 200023,
11
+ "<|user|>": 200021
12
+ }
checkpoint-500/chat_template.jinja ADDED
@@ -0,0 +1 @@
 
 
1
+ {% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}
checkpoint-500/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21ca6a38213dbcecea4636acdf726c0ed77661235c93517b024209af6720a7b3
3
+ size 18884901
checkpoint-500/global_step500/mp_rank_00_model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8efbc708dea4e851cdfb5141b66e24b3da83a4bb325a4f22c8b93ed262b391ae
3
+ size 1232414501
checkpoint-500/latest ADDED
@@ -0,0 +1 @@
 
 
1
+ global_step500
checkpoint-500/merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-500/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:898a69b8bed5380622ff1fcead44b9c1d64d1f171aa4d028382c05e4361e8056
3
+ size 14709
checkpoint-500/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c5bed181fb8edf311a86cbd8e7d9bfec1a9c122d64a72655e253323e7205104
3
+ size 1465
checkpoint-500/special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|endoftext|>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<|endoftext|>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<|endoftext|>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
checkpoint-500/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:382cc235b56c725945e149cc25f191da667c836655efd0857b004320e90e91ea
3
+ size 15524095
checkpoint-500/tokenizer_config.json ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": false,
5
+ "added_tokens_decoder": {
6
+ "199999": {
7
+ "content": "<|endoftext|>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "200018": {
15
+ "content": "<|endofprompt|>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "200019": {
23
+ "content": "<|assistant|>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": true,
27
+ "single_word": false,
28
+ "special": true
29
+ },
30
+ "200020": {
31
+ "content": "<|end|>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": true,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
+ "200021": {
39
+ "content": "<|user|>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": true,
43
+ "single_word": false,
44
+ "special": true
45
+ },
46
+ "200022": {
47
+ "content": "<|system|>",
48
+ "lstrip": false,
49
+ "normalized": false,
50
+ "rstrip": true,
51
+ "single_word": false,
52
+ "special": true
53
+ },
54
+ "200023": {
55
+ "content": "<|tool|>",
56
+ "lstrip": false,
57
+ "normalized": false,
58
+ "rstrip": true,
59
+ "single_word": false,
60
+ "special": false
61
+ },
62
+ "200024": {
63
+ "content": "<|/tool|>",
64
+ "lstrip": false,
65
+ "normalized": false,
66
+ "rstrip": true,
67
+ "single_word": false,
68
+ "special": false
69
+ },
70
+ "200025": {
71
+ "content": "<|tool_call|>",
72
+ "lstrip": false,
73
+ "normalized": false,
74
+ "rstrip": true,
75
+ "single_word": false,
76
+ "special": false
77
+ },
78
+ "200026": {
79
+ "content": "<|/tool_call|>",
80
+ "lstrip": false,
81
+ "normalized": false,
82
+ "rstrip": true,
83
+ "single_word": false,
84
+ "special": false
85
+ },
86
+ "200027": {
87
+ "content": "<|tool_response|>",
88
+ "lstrip": false,
89
+ "normalized": false,
90
+ "rstrip": true,
91
+ "single_word": false,
92
+ "special": false
93
+ },
94
+ "200028": {
95
+ "content": "<|tag|>",
96
+ "lstrip": false,
97
+ "normalized": false,
98
+ "rstrip": true,
99
+ "single_word": false,
100
+ "special": true
101
+ }
102
+ },
103
+ "bos_token": "<|endoftext|>",
104
+ "clean_up_tokenization_spaces": false,
105
+ "eos_token": "<|endoftext|>",
106
+ "extra_special_tokens": {},
107
+ "model_max_length": 131072,
108
+ "pad_token": "<|endoftext|>",
109
+ "tokenizer_class": "GPT2Tokenizer",
110
+ "unk_token": "<|endoftext|>"
111
+ }
checkpoint-500/tokens_state. ADDED
@@ -0,0 +1 @@
 
 
1
+ {"total": 8036352, "trainable": 3634339}
checkpoint-500/trainer_state.json ADDED
@@ -0,0 +1,1566 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 38.470588235294116,
6
+ "eval_steps": 50,
7
+ "global_step": 500,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0,
14
+ "eval_loss": 2.1261069774627686,
15
+ "eval_ppl": 8.38217,
16
+ "eval_runtime": 51.9298,
17
+ "eval_samples_per_second": 3.851,
18
+ "eval_steps_per_second": 1.926,
19
+ "memory/device_reserved (GiB)": 15.32,
20
+ "memory/max_active (GiB)": 14.81,
21
+ "memory/max_allocated (GiB)": 14.81,
22
+ "step": 0
23
+ },
24
+ {
25
+ "epoch": 0.39215686274509803,
26
+ "grad_norm": 0.27175864577293396,
27
+ "learning_rate": 4.000000000000001e-06,
28
+ "loss": 6.0524,
29
+ "memory/device_reserved (GiB)": 31.8,
30
+ "memory/max_active (GiB)": 30.03,
31
+ "memory/max_allocated (GiB)": 30.03,
32
+ "ppl": 425.13212,
33
+ "step": 5,
34
+ "tokens/total": 81920,
35
+ "tokens/train_per_sec_per_gpu": 46.17,
36
+ "tokens/trainable": 36899
37
+ },
38
+ {
39
+ "epoch": 0.7843137254901961,
40
+ "grad_norm": 0.26692718267440796,
41
+ "learning_rate": 9e-06,
42
+ "loss": 5.992,
43
+ "memory/device_reserved (GiB)": 31.8,
44
+ "memory/max_active (GiB)": 30.03,
45
+ "memory/max_allocated (GiB)": 30.03,
46
+ "ppl": 400.21424,
47
+ "step": 10,
48
+ "tokens/total": 163840,
49
+ "tokens/train_per_sec_per_gpu": 47.3,
50
+ "tokens/trainable": 74260
51
+ },
52
+ {
53
+ "epoch": 1.156862745098039,
54
+ "grad_norm": 0.306497722864151,
55
+ "learning_rate": 1.4e-05,
56
+ "loss": 5.9906,
57
+ "memory/device_reserved (GiB)": 31.8,
58
+ "memory/max_active (GiB)": 30.03,
59
+ "memory/max_allocated (GiB)": 30.03,
60
+ "ppl": 399.65433,
61
+ "step": 15,
62
+ "tokens/total": 241664,
63
+ "tokens/train_per_sec_per_gpu": 48.66,
64
+ "tokens/trainable": 109275
65
+ },
66
+ {
67
+ "epoch": 1.5490196078431373,
68
+ "grad_norm": 0.3827843964099884,
69
+ "learning_rate": 1.9e-05,
70
+ "loss": 6.0295,
71
+ "memory/device_reserved (GiB)": 31.8,
72
+ "memory/max_active (GiB)": 30.03,
73
+ "memory/max_allocated (GiB)": 30.03,
74
+ "ppl": 415.50722,
75
+ "step": 20,
76
+ "tokens/total": 323584,
77
+ "tokens/train_per_sec_per_gpu": 46.01,
78
+ "tokens/trainable": 146594
79
+ },
80
+ {
81
+ "epoch": 1.9411764705882353,
82
+ "grad_norm": 0.46802398562431335,
83
+ "learning_rate": 1.9998010727705237e-05,
84
+ "loss": 5.9689,
85
+ "memory/device_reserved (GiB)": 31.8,
86
+ "memory/max_active (GiB)": 30.03,
87
+ "memory/max_allocated (GiB)": 30.03,
88
+ "ppl": 391.07525,
89
+ "step": 25,
90
+ "tokens/total": 405504,
91
+ "tokens/train_per_sec_per_gpu": 45.66,
92
+ "tokens/trainable": 183790
93
+ },
94
+ {
95
+ "epoch": 2.313725490196078,
96
+ "grad_norm": 0.5154851675033569,
97
+ "learning_rate": 1.9989930665413148e-05,
98
+ "loss": 5.8882,
99
+ "memory/device_reserved (GiB)": 31.8,
100
+ "memory/max_active (GiB)": 30.03,
101
+ "memory/max_allocated (GiB)": 30.03,
102
+ "ppl": 360.75534,
103
+ "step": 30,
104
+ "tokens/total": 483328,
105
+ "tokens/train_per_sec_per_gpu": 45.15,
106
+ "tokens/trainable": 218443
107
+ },
108
+ {
109
+ "epoch": 2.7058823529411766,
110
+ "grad_norm": 0.6076357364654541,
111
+ "learning_rate": 1.9975640502598243e-05,
112
+ "loss": 5.7916,
113
+ "memory/device_reserved (GiB)": 31.8,
114
+ "memory/max_active (GiB)": 30.03,
115
+ "memory/max_allocated (GiB)": 30.03,
116
+ "ppl": 327.53666,
117
+ "step": 35,
118
+ "tokens/total": 565248,
119
+ "tokens/train_per_sec_per_gpu": 47.92,
120
+ "tokens/trainable": 255824
121
+ },
122
+ {
123
+ "epoch": 3.0784313725490198,
124
+ "grad_norm": 0.6261164546012878,
125
+ "learning_rate": 1.995514912254015e-05,
126
+ "loss": 5.7018,
127
+ "memory/device_reserved (GiB)": 31.8,
128
+ "memory/max_active (GiB)": 30.03,
129
+ "memory/max_allocated (GiB)": 30.03,
130
+ "ppl": 299.40585,
131
+ "step": 40,
132
+ "tokens/total": 643072,
133
+ "tokens/train_per_sec_per_gpu": 45.74,
134
+ "tokens/trainable": 290916
135
+ },
136
+ {
137
+ "epoch": 3.4705882352941178,
138
+ "grad_norm": 0.7209100127220154,
139
+ "learning_rate": 1.9928469263418376e-05,
140
+ "loss": 5.5957,
141
+ "memory/device_reserved (GiB)": 31.8,
142
+ "memory/max_active (GiB)": 30.03,
143
+ "memory/max_allocated (GiB)": 30.03,
144
+ "ppl": 269.26607,
145
+ "step": 45,
146
+ "tokens/total": 724992,
147
+ "tokens/train_per_sec_per_gpu": 46.62,
148
+ "tokens/trainable": 328157
149
+ },
150
+ {
151
+ "epoch": 3.8627450980392157,
152
+ "grad_norm": 0.6933228373527527,
153
+ "learning_rate": 1.9895617510393773e-05,
154
+ "loss": 5.5167,
155
+ "memory/device_reserved (GiB)": 31.8,
156
+ "memory/max_active (GiB)": 30.03,
157
+ "memory/max_allocated (GiB)": 30.03,
158
+ "ppl": 248.8126,
159
+ "step": 50,
160
+ "tokens/total": 806912,
161
+ "tokens/train_per_sec_per_gpu": 45.37,
162
+ "tokens/trainable": 365292
163
+ },
164
+ {
165
+ "epoch": 3.8627450980392157,
166
+ "eval_loss": 2.1055686473846436,
167
+ "eval_ppl": 8.21177,
168
+ "eval_runtime": 52.5301,
169
+ "eval_samples_per_second": 3.807,
170
+ "eval_steps_per_second": 1.904,
171
+ "memory/device_reserved (GiB)": 31.8,
172
+ "memory/max_active (GiB)": 14.82,
173
+ "memory/max_allocated (GiB)": 14.82,
174
+ "step": 50
175
+ },
176
+ {
177
+ "epoch": 4.235294117647059,
178
+ "grad_norm": 0.7358637452125549,
179
+ "learning_rate": 1.985661428529863e-05,
180
+ "loss": 5.343,
181
+ "memory/device_reserved (GiB)": 31.82,
182
+ "memory/max_active (GiB)": 30.03,
183
+ "memory/max_allocated (GiB)": 30.03,
184
+ "ppl": 209.13919,
185
+ "step": 55,
186
+ "tokens/total": 884736,
187
+ "tokens/train_per_sec_per_gpu": 44.66,
188
+ "tokens/trainable": 399785
189
+ },
190
+ {
191
+ "epoch": 4.627450980392156,
192
+ "grad_norm": 0.6546111702919006,
193
+ "learning_rate": 1.9811483833941726e-05,
194
+ "loss": 5.2353,
195
+ "memory/device_reserved (GiB)": 31.82,
196
+ "memory/max_active (GiB)": 30.03,
197
+ "memory/max_allocated (GiB)": 30.03,
198
+ "ppl": 187.78543,
199
+ "step": 60,
200
+ "tokens/total": 966656,
201
+ "tokens/train_per_sec_per_gpu": 42.79,
202
+ "tokens/trainable": 437478
203
+ },
204
+ {
205
+ "epoch": 5.0,
206
+ "grad_norm": 0.7404906749725342,
207
+ "learning_rate": 1.9760254211036245e-05,
208
+ "loss": 5.1051,
209
+ "memory/device_reserved (GiB)": 31.82,
210
+ "memory/max_active (GiB)": 30.03,
211
+ "memory/max_allocated (GiB)": 30.03,
212
+ "ppl": 164.86056,
213
+ "step": 65,
214
+ "tokens/total": 1044480,
215
+ "tokens/train_per_sec_per_gpu": 41.51,
216
+ "tokens/trainable": 472345
217
+ },
218
+ {
219
+ "epoch": 5.392156862745098,
220
+ "grad_norm": 0.6113602519035339,
221
+ "learning_rate": 1.9702957262759964e-05,
222
+ "loss": 4.9746,
223
+ "memory/device_reserved (GiB)": 31.82,
224
+ "memory/max_active (GiB)": 30.03,
225
+ "memory/max_allocated (GiB)": 30.03,
226
+ "ppl": 144.69094,
227
+ "step": 70,
228
+ "tokens/total": 1126400,
229
+ "tokens/train_per_sec_per_gpu": 47.33,
230
+ "tokens/trainable": 509900
231
+ },
232
+ {
233
+ "epoch": 5.784313725490196,
234
+ "grad_norm": 0.5721343755722046,
235
+ "learning_rate": 1.9639628606958535e-05,
236
+ "loss": 4.8985,
237
+ "memory/device_reserved (GiB)": 31.82,
238
+ "memory/max_active (GiB)": 30.03,
239
+ "memory/max_allocated (GiB)": 30.03,
240
+ "ppl": 134.0885,
241
+ "step": 75,
242
+ "tokens/total": 1208320,
243
+ "tokens/train_per_sec_per_gpu": 46.45,
244
+ "tokens/trainable": 547077
245
+ },
246
+ {
247
+ "epoch": 6.1568627450980395,
248
+ "grad_norm": 0.6451320052146912,
249
+ "learning_rate": 1.9570307611004124e-05,
250
+ "loss": 4.7955,
251
+ "memory/device_reserved (GiB)": 31.82,
252
+ "memory/max_active (GiB)": 30.03,
253
+ "memory/max_allocated (GiB)": 30.03,
254
+ "ppl": 120.96485,
255
+ "step": 80,
256
+ "tokens/total": 1286144,
257
+ "tokens/train_per_sec_per_gpu": 47.54,
258
+ "tokens/trainable": 581824
259
+ },
260
+ {
261
+ "epoch": 6.549019607843137,
262
+ "grad_norm": 0.6520641446113586,
263
+ "learning_rate": 1.9495037367323264e-05,
264
+ "loss": 4.7514,
265
+ "memory/device_reserved (GiB)": 31.82,
266
+ "memory/max_active (GiB)": 30.03,
267
+ "memory/max_allocated (GiB)": 30.03,
268
+ "ppl": 115.74622,
269
+ "step": 85,
270
+ "tokens/total": 1368064,
271
+ "tokens/train_per_sec_per_gpu": 44.41,
272
+ "tokens/trainable": 618867
273
+ },
274
+ {
275
+ "epoch": 6.9411764705882355,
276
+ "grad_norm": 0.6384134888648987,
277
+ "learning_rate": 1.9413864666609036e-05,
278
+ "loss": 4.6298,
279
+ "memory/device_reserved (GiB)": 31.82,
280
+ "memory/max_active (GiB)": 30.03,
281
+ "memory/max_allocated (GiB)": 30.03,
282
+ "ppl": 102.49356,
283
+ "step": 90,
284
+ "tokens/total": 1449984,
285
+ "tokens/train_per_sec_per_gpu": 44.14,
286
+ "tokens/trainable": 656298
287
+ },
288
+ {
289
+ "epoch": 7.313725490196078,
290
+ "grad_norm": 0.6989919543266296,
291
+ "learning_rate": 1.9326839968734278e-05,
292
+ "loss": 4.5767,
293
+ "memory/device_reserved (GiB)": 31.82,
294
+ "memory/max_active (GiB)": 30.03,
295
+ "memory/max_allocated (GiB)": 30.03,
296
+ "ppl": 97.19313,
297
+ "step": 95,
298
+ "tokens/total": 1527808,
299
+ "tokens/train_per_sec_per_gpu": 45.78,
300
+ "tokens/trainable": 691156
301
+ },
302
+ {
303
+ "epoch": 7.705882352941177,
304
+ "grad_norm": 0.630689263343811,
305
+ "learning_rate": 1.9234017371383946e-05,
306
+ "loss": 4.5059,
307
+ "memory/device_reserved (GiB)": 31.82,
308
+ "memory/max_active (GiB)": 30.03,
309
+ "memory/max_allocated (GiB)": 30.03,
310
+ "ppl": 90.5498,
311
+ "step": 100,
312
+ "tokens/total": 1609728,
313
+ "tokens/train_per_sec_per_gpu": 46.37,
314
+ "tokens/trainable": 727983
315
+ },
316
+ {
317
+ "epoch": 7.705882352941177,
318
+ "eval_loss": 2.0381510257720947,
319
+ "eval_ppl": 7.6764,
320
+ "eval_runtime": 52.3831,
321
+ "eval_samples_per_second": 3.818,
322
+ "eval_steps_per_second": 1.909,
323
+ "memory/device_reserved (GiB)": 31.82,
324
+ "memory/max_active (GiB)": 14.82,
325
+ "memory/max_allocated (GiB)": 14.82,
326
+ "step": 100
327
+ },
328
+ {
329
+ "epoch": 8.07843137254902,
330
+ "grad_norm": 0.6709732413291931,
331
+ "learning_rate": 1.913545457642601e-05,
332
+ "loss": 4.3844,
333
+ "memory/device_reserved (GiB)": 31.82,
334
+ "memory/max_active (GiB)": 30.03,
335
+ "memory/max_allocated (GiB)": 30.03,
336
+ "ppl": 80.19009,
337
+ "step": 105,
338
+ "tokens/total": 1687552,
339
+ "tokens/train_per_sec_per_gpu": 45.03,
340
+ "tokens/trainable": 763205
341
+ },
342
+ {
343
+ "epoch": 8.470588235294118,
344
+ "grad_norm": 0.6814770698547363,
345
+ "learning_rate": 1.903121285404192e-05,
346
+ "loss": 4.3581,
347
+ "memory/device_reserved (GiB)": 31.82,
348
+ "memory/max_active (GiB)": 30.03,
349
+ "memory/max_allocated (GiB)": 30.03,
350
+ "ppl": 78.10859,
351
+ "step": 110,
352
+ "tokens/total": 1769472,
353
+ "tokens/train_per_sec_per_gpu": 45.04,
354
+ "tokens/trainable": 801058
355
+ },
356
+ {
357
+ "epoch": 8.862745098039216,
358
+ "grad_norm": 0.7326321005821228,
359
+ "learning_rate": 1.8921357004638837e-05,
360
+ "loss": 4.2923,
361
+ "memory/device_reserved (GiB)": 31.82,
362
+ "memory/max_active (GiB)": 30.03,
363
+ "memory/max_allocated (GiB)": 30.03,
364
+ "ppl": 73.13448,
365
+ "step": 115,
366
+ "tokens/total": 1851392,
367
+ "tokens/train_per_sec_per_gpu": 44.88,
368
+ "tokens/trainable": 837904
369
+ },
370
+ {
371
+ "epoch": 9.235294117647058,
372
+ "grad_norm": 0.7615563869476318,
373
+ "learning_rate": 1.880595531856738e-05,
374
+ "loss": 4.1865,
375
+ "memory/device_reserved (GiB)": 31.82,
376
+ "memory/max_active (GiB)": 30.03,
377
+ "memory/max_allocated (GiB)": 30.03,
378
+ "ppl": 65.79212,
379
+ "step": 120,
380
+ "tokens/total": 1929216,
381
+ "tokens/train_per_sec_per_gpu": 48.18,
382
+ "tokens/trainable": 872791
383
+ },
384
+ {
385
+ "epoch": 9.627450980392156,
386
+ "grad_norm": 0.6976982951164246,
387
+ "learning_rate": 1.868507953366989e-05,
388
+ "loss": 4.157,
389
+ "memory/device_reserved (GiB)": 31.82,
390
+ "memory/max_active (GiB)": 30.03,
391
+ "memory/max_allocated (GiB)": 30.03,
392
+ "ppl": 63.8796,
393
+ "step": 125,
394
+ "tokens/total": 2011136,
395
+ "tokens/train_per_sec_per_gpu": 48.15,
396
+ "tokens/trainable": 910332
397
+ },
398
+ {
399
+ "epoch": 10.0,
400
+ "grad_norm": 0.9011051058769226,
401
+ "learning_rate": 1.855880479068559e-05,
402
+ "loss": 4.0724,
403
+ "memory/device_reserved (GiB)": 31.82,
404
+ "memory/max_active (GiB)": 30.03,
405
+ "memory/max_allocated (GiB)": 30.03,
406
+ "ppl": 58.69767,
407
+ "step": 130,
408
+ "tokens/total": 2088960,
409
+ "tokens/train_per_sec_per_gpu": 44.4,
410
+ "tokens/trainable": 944690
411
+ },
412
+ {
413
+ "epoch": 10.392156862745098,
414
+ "grad_norm": 0.7039150595664978,
415
+ "learning_rate": 1.8427209586540392e-05,
416
+ "loss": 4.0452,
417
+ "memory/device_reserved (GiB)": 31.82,
418
+ "memory/max_active (GiB)": 30.03,
419
+ "memory/max_allocated (GiB)": 30.03,
420
+ "ppl": 57.12261,
421
+ "step": 135,
422
+ "tokens/total": 2170880,
423
+ "tokens/train_per_sec_per_gpu": 44.67,
424
+ "tokens/trainable": 981807
425
+ },
426
+ {
427
+ "epoch": 10.784313725490197,
428
+ "grad_norm": 0.7844482660293579,
429
+ "learning_rate": 1.8290375725550417e-05,
430
+ "loss": 3.9809,
431
+ "memory/device_reserved (GiB)": 31.82,
432
+ "memory/max_active (GiB)": 30.03,
433
+ "memory/max_allocated (GiB)": 30.03,
434
+ "ppl": 53.56522,
435
+ "step": 140,
436
+ "tokens/total": 2252800,
437
+ "tokens/train_per_sec_per_gpu": 52.01,
438
+ "tokens/trainable": 1019070
439
+ },
440
+ {
441
+ "epoch": 11.156862745098039,
442
+ "grad_norm": 0.8043951392173767,
443
+ "learning_rate": 1.8148388268569453e-05,
444
+ "loss": 3.8289,
445
+ "memory/device_reserved (GiB)": 31.82,
446
+ "memory/max_active (GiB)": 30.03,
447
+ "memory/max_allocated (GiB)": 30.03,
448
+ "ppl": 46.0119,
449
+ "step": 145,
450
+ "tokens/total": 2330624,
451
+ "tokens/train_per_sec_per_gpu": 42.58,
452
+ "tokens/trainable": 1053699
453
+ },
454
+ {
455
+ "epoch": 11.549019607843137,
456
+ "grad_norm": 0.7718515992164612,
457
+ "learning_rate": 1.8001335480112067e-05,
458
+ "loss": 3.8251,
459
+ "memory/device_reserved (GiB)": 31.82,
460
+ "memory/max_active (GiB)": 30.03,
461
+ "memory/max_allocated (GiB)": 30.03,
462
+ "ppl": 45.83738,
463
+ "step": 150,
464
+ "tokens/total": 2412544,
465
+ "tokens/train_per_sec_per_gpu": 45.26,
466
+ "tokens/trainable": 1091200
467
+ },
468
+ {
469
+ "epoch": 11.549019607843137,
470
+ "eval_loss": 1.980879783630371,
471
+ "eval_ppl": 7.24912,
472
+ "eval_runtime": 52.4263,
473
+ "eval_samples_per_second": 3.815,
474
+ "eval_steps_per_second": 1.907,
475
+ "memory/device_reserved (GiB)": 31.82,
476
+ "memory/max_active (GiB)": 14.82,
477
+ "memory/max_allocated (GiB)": 14.82,
478
+ "step": 150
479
+ },
480
+ {
481
+ "epoch": 11.941176470588236,
482
+ "grad_norm": 0.8142716288566589,
483
+ "learning_rate": 1.7849308773485226e-05,
484
+ "loss": 3.7862,
485
+ "memory/device_reserved (GiB)": 31.82,
486
+ "memory/max_active (GiB)": 30.03,
487
+ "memory/max_allocated (GiB)": 30.03,
488
+ "ppl": 44.08855,
489
+ "step": 155,
490
+ "tokens/total": 2494464,
491
+ "tokens/train_per_sec_per_gpu": 46.91,
492
+ "tokens/trainable": 1128496
493
+ },
494
+ {
495
+ "epoch": 12.313725490196079,
496
+ "grad_norm": 0.7591995596885681,
497
+ "learning_rate": 1.769240265396249e-05,
498
+ "loss": 3.7156,
499
+ "memory/device_reserved (GiB)": 31.82,
500
+ "memory/max_active (GiB)": 30.03,
501
+ "memory/max_allocated (GiB)": 30.03,
502
+ "ppl": 41.08323,
503
+ "step": 160,
504
+ "tokens/total": 2572288,
505
+ "tokens/train_per_sec_per_gpu": 45.85,
506
+ "tokens/trainable": 1164008
507
+ },
508
+ {
509
+ "epoch": 12.705882352941176,
510
+ "grad_norm": 0.7124404311180115,
511
+ "learning_rate": 1.7530714660036112e-05,
512
+ "loss": 3.6955,
513
+ "memory/device_reserved (GiB)": 31.82,
514
+ "memory/max_active (GiB)": 30.03,
515
+ "memory/max_allocated (GiB)": 30.03,
516
+ "ppl": 40.2657,
517
+ "step": 165,
518
+ "tokens/total": 2654208,
519
+ "tokens/train_per_sec_per_gpu": 43.85,
520
+ "tokens/trainable": 1200877
521
+ },
522
+ {
523
+ "epoch": 13.07843137254902,
524
+ "grad_norm": 0.7880769371986389,
525
+ "learning_rate": 1.736434530278362e-05,
526
+ "loss": 3.6283,
527
+ "memory/device_reserved (GiB)": 31.82,
528
+ "memory/max_active (GiB)": 30.03,
529
+ "memory/max_allocated (GiB)": 30.03,
530
+ "ppl": 37.64876,
531
+ "step": 170,
532
+ "tokens/total": 2732032,
533
+ "tokens/train_per_sec_per_gpu": 47.57,
534
+ "tokens/trainable": 1235550
535
+ },
536
+ {
537
+ "epoch": 13.470588235294118,
538
+ "grad_norm": 0.7571734189987183,
539
+ "learning_rate": 1.7193398003386514e-05,
540
+ "loss": 3.5842,
541
+ "memory/device_reserved (GiB)": 31.82,
542
+ "memory/max_active (GiB)": 30.03,
543
+ "memory/max_allocated (GiB)": 30.03,
544
+ "ppl": 36.02453,
545
+ "step": 175,
546
+ "tokens/total": 2813952,
547
+ "tokens/train_per_sec_per_gpu": 44.96,
548
+ "tokens/trainable": 1272589
549
+ },
550
+ {
551
+ "epoch": 13.862745098039216,
552
+ "grad_norm": 0.9505294561386108,
553
+ "learning_rate": 1.7017979028839918e-05,
554
+ "loss": 3.555,
555
+ "memory/device_reserved (GiB)": 31.82,
556
+ "memory/max_active (GiB)": 30.03,
557
+ "memory/max_allocated (GiB)": 30.03,
558
+ "ppl": 34.98782,
559
+ "step": 180,
560
+ "tokens/total": 2895872,
561
+ "tokens/train_per_sec_per_gpu": 43.68,
562
+ "tokens/trainable": 1309972
563
+ },
564
+ {
565
+ "epoch": 14.235294117647058,
566
+ "grad_norm": 1.3748526573181152,
567
+ "learning_rate": 1.68381974258932e-05,
568
+ "loss": 3.5266,
569
+ "memory/device_reserved (GiB)": 31.82,
570
+ "memory/max_active (GiB)": 30.03,
571
+ "memory/max_allocated (GiB)": 30.03,
572
+ "ppl": 34.00814,
573
+ "step": 185,
574
+ "tokens/total": 2973696,
575
+ "tokens/train_per_sec_per_gpu": 42.81,
576
+ "tokens/trainable": 1344786
577
+ },
578
+ {
579
+ "epoch": 14.627450980392156,
580
+ "grad_norm": 0.7306499481201172,
581
+ "learning_rate": 1.6654164953262614e-05,
582
+ "loss": 3.4729,
583
+ "memory/device_reserved (GiB)": 31.82,
584
+ "memory/max_active (GiB)": 30.03,
585
+ "memory/max_allocated (GiB)": 30.03,
586
+ "ppl": 32.23007,
587
+ "step": 190,
588
+ "tokens/total": 3055616,
589
+ "tokens/train_per_sec_per_gpu": 43.64,
590
+ "tokens/trainable": 1382206
591
+ },
592
+ {
593
+ "epoch": 15.0,
594
+ "grad_norm": 1.0910062789916992,
595
+ "learning_rate": 1.6465996012157996e-05,
596
+ "loss": 3.3893,
597
+ "memory/device_reserved (GiB)": 31.82,
598
+ "memory/max_active (GiB)": 30.03,
599
+ "memory/max_allocated (GiB)": 30.03,
600
+ "ppl": 29.64519,
601
+ "step": 195,
602
+ "tokens/total": 3133440,
603
+ "tokens/train_per_sec_per_gpu": 41.36,
604
+ "tokens/trainable": 1417035
605
+ },
606
+ {
607
+ "epoch": 15.392156862745098,
608
+ "grad_norm": 0.8951497673988342,
609
+ "learning_rate": 1.6273807575166927e-05,
610
+ "loss": 3.4152,
611
+ "memory/device_reserved (GiB)": 31.82,
612
+ "memory/max_active (GiB)": 30.03,
613
+ "memory/max_allocated (GiB)": 30.03,
614
+ "ppl": 30.42303,
615
+ "step": 200,
616
+ "tokens/total": 3215360,
617
+ "tokens/train_per_sec_per_gpu": 45.47,
618
+ "tokens/trainable": 1453690
619
+ },
620
+ {
621
+ "epoch": 15.392156862745098,
622
+ "eval_loss": 1.934309720993042,
623
+ "eval_ppl": 6.91927,
624
+ "eval_runtime": 52.4116,
625
+ "eval_samples_per_second": 3.816,
626
+ "eval_steps_per_second": 1.908,
627
+ "memory/device_reserved (GiB)": 31.82,
628
+ "memory/max_active (GiB)": 14.82,
629
+ "memory/max_allocated (GiB)": 14.82,
630
+ "step": 200
631
+ },
632
+ {
633
+ "epoch": 15.784313725490197,
634
+ "grad_norm": 0.8389677405357361,
635
+ "learning_rate": 1.6077719113540303e-05,
636
+ "loss": 3.3746,
637
+ "memory/device_reserved (GiB)": 31.82,
638
+ "memory/max_active (GiB)": 30.03,
639
+ "memory/max_allocated (GiB)": 30.03,
640
+ "ppl": 29.2126,
641
+ "step": 205,
642
+ "tokens/total": 3297280,
643
+ "tokens/train_per_sec_per_gpu": 46.15,
644
+ "tokens/trainable": 1491448
645
+ },
646
+ {
647
+ "epoch": 16.15686274509804,
648
+ "grad_norm": 0.9337797164916992,
649
+ "learning_rate": 1.5877852522924733e-05,
650
+ "loss": 3.3149,
651
+ "memory/device_reserved (GiB)": 31.82,
652
+ "memory/max_active (GiB)": 30.03,
653
+ "memory/max_allocated (GiB)": 30.03,
654
+ "ppl": 27.51964,
655
+ "step": 210,
656
+ "tokens/total": 3375104,
657
+ "tokens/train_per_sec_per_gpu": 45.59,
658
+ "tokens/trainable": 1526395
659
+ },
660
+ {
661
+ "epoch": 16.54901960784314,
662
+ "grad_norm": 0.8584134578704834,
663
+ "learning_rate": 1.567433204758782e-05,
664
+ "loss": 3.3092,
665
+ "memory/device_reserved (GiB)": 31.82,
666
+ "memory/max_active (GiB)": 30.03,
667
+ "memory/max_allocated (GiB)": 30.03,
668
+ "ppl": 27.36323,
669
+ "step": 215,
670
+ "tokens/total": 3457024,
671
+ "tokens/train_per_sec_per_gpu": 45.82,
672
+ "tokens/trainable": 1563348
673
+ },
674
+ {
675
+ "epoch": 16.941176470588236,
676
+ "grad_norm": 0.8377186059951782,
677
+ "learning_rate": 1.5467284203183437e-05,
678
+ "loss": 3.2949,
679
+ "memory/device_reserved (GiB)": 31.82,
680
+ "memory/max_active (GiB)": 30.03,
681
+ "memory/max_allocated (GiB)": 30.03,
682
+ "ppl": 26.97472,
683
+ "step": 220,
684
+ "tokens/total": 3538944,
685
+ "tokens/train_per_sec_per_gpu": 44.83,
686
+ "tokens/trainable": 1600934
687
+ },
688
+ {
689
+ "epoch": 17.313725490196077,
690
+ "grad_norm": 0.814207911491394,
691
+ "learning_rate": 1.5256837698105047e-05,
692
+ "loss": 3.2277,
693
+ "memory/device_reserved (GiB)": 31.82,
694
+ "memory/max_active (GiB)": 30.03,
695
+ "memory/max_allocated (GiB)": 30.03,
696
+ "ppl": 25.22158,
697
+ "step": 225,
698
+ "tokens/total": 3616768,
699
+ "tokens/train_per_sec_per_gpu": 46.04,
700
+ "tokens/trainable": 1636161
701
+ },
702
+ {
703
+ "epoch": 17.705882352941178,
704
+ "grad_norm": 0.8848176598548889,
705
+ "learning_rate": 1.5043123353475944e-05,
706
+ "loss": 3.2177,
707
+ "memory/device_reserved (GiB)": 31.82,
708
+ "memory/max_active (GiB)": 30.03,
709
+ "memory/max_allocated (GiB)": 30.03,
710
+ "ppl": 24.97062,
711
+ "step": 230,
712
+ "tokens/total": 3698688,
713
+ "tokens/train_per_sec_per_gpu": 45.18,
714
+ "tokens/trainable": 1673301
715
+ },
716
+ {
717
+ "epoch": 18.07843137254902,
718
+ "grad_norm": 0.847163736820221,
719
+ "learning_rate": 1.482627402182611e-05,
720
+ "loss": 3.2007,
721
+ "memory/device_reserved (GiB)": 31.82,
722
+ "memory/max_active (GiB)": 30.03,
723
+ "memory/max_allocated (GiB)": 30.03,
724
+ "ppl": 24.54971,
725
+ "step": 235,
726
+ "tokens/total": 3776512,
727
+ "tokens/train_per_sec_per_gpu": 42.62,
728
+ "tokens/trainable": 1707898
729
+ },
730
+ {
731
+ "epoch": 18.470588235294116,
732
+ "grad_norm": 0.8863293528556824,
733
+ "learning_rate": 1.4606424504506325e-05,
734
+ "loss": 3.1473,
735
+ "memory/device_reserved (GiB)": 31.82,
736
+ "memory/max_active (GiB)": 30.03,
737
+ "memory/max_allocated (GiB)": 30.03,
738
+ "ppl": 23.27314,
739
+ "step": 240,
740
+ "tokens/total": 3858432,
741
+ "tokens/train_per_sec_per_gpu": 46.4,
742
+ "tokens/trainable": 1745243
743
+ },
744
+ {
745
+ "epoch": 18.862745098039216,
746
+ "grad_norm": 0.8567431569099426,
747
+ "learning_rate": 1.4383711467890776e-05,
748
+ "loss": 3.1299,
749
+ "memory/device_reserved (GiB)": 31.82,
750
+ "memory/max_active (GiB)": 30.03,
751
+ "memory/max_allocated (GiB)": 30.03,
752
+ "ppl": 22.87169,
753
+ "step": 245,
754
+ "tokens/total": 3940352,
755
+ "tokens/train_per_sec_per_gpu": 46.63,
756
+ "tokens/trainable": 1782188
757
+ },
758
+ {
759
+ "epoch": 19.235294117647058,
760
+ "grad_norm": 0.8873140215873718,
761
+ "learning_rate": 1.415827335842033e-05,
762
+ "loss": 3.1617,
763
+ "memory/device_reserved (GiB)": 31.82,
764
+ "memory/max_active (GiB)": 30.03,
765
+ "memory/max_allocated (GiB)": 30.03,
766
+ "ppl": 23.6107,
767
+ "step": 250,
768
+ "tokens/total": 4018176,
769
+ "tokens/train_per_sec_per_gpu": 46.02,
770
+ "tokens/trainable": 1817232
771
+ },
772
+ {
773
+ "epoch": 19.235294117647058,
774
+ "eval_loss": 1.8731095790863037,
775
+ "eval_ppl": 6.5085,
776
+ "eval_runtime": 52.2712,
777
+ "eval_samples_per_second": 3.826,
778
+ "eval_steps_per_second": 1.913,
779
+ "memory/device_reserved (GiB)": 31.82,
780
+ "memory/max_active (GiB)": 14.82,
781
+ "memory/max_allocated (GiB)": 14.82,
782
+ "step": 250
783
+ },
784
+ {
785
+ "epoch": 19.627450980392158,
786
+ "grad_norm": 0.8352880477905273,
787
+ "learning_rate": 1.3930250316539237e-05,
788
+ "loss": 3.0786,
789
+ "memory/device_reserved (GiB)": 31.82,
790
+ "memory/max_active (GiB)": 30.03,
791
+ "memory/max_allocated (GiB)": 30.03,
792
+ "ppl": 21.72796,
793
+ "step": 255,
794
+ "tokens/total": 4100096,
795
+ "tokens/train_per_sec_per_gpu": 45.37,
796
+ "tokens/trainable": 1854866
797
+ },
798
+ {
799
+ "epoch": 20.0,
800
+ "grad_norm": 1.1509190797805786,
801
+ "learning_rate": 1.3699784089578791e-05,
802
+ "loss": 3.0425,
803
+ "memory/device_reserved (GiB)": 31.82,
804
+ "memory/max_active (GiB)": 30.03,
805
+ "memory/max_allocated (GiB)": 30.03,
806
+ "ppl": 20.95757,
807
+ "step": 260,
808
+ "tokens/total": 4177920,
809
+ "tokens/train_per_sec_per_gpu": 41.77,
810
+ "tokens/trainable": 1889380
811
+ },
812
+ {
813
+ "epoch": 20.392156862745097,
814
+ "grad_norm": 0.8712507486343384,
815
+ "learning_rate": 1.3467017943642074e-05,
816
+ "loss": 3.0706,
817
+ "memory/device_reserved (GiB)": 31.82,
818
+ "memory/max_active (GiB)": 30.03,
819
+ "memory/max_allocated (GiB)": 30.03,
820
+ "ppl": 21.55483,
821
+ "step": 265,
822
+ "tokens/total": 4259840,
823
+ "tokens/train_per_sec_per_gpu": 46.68,
824
+ "tokens/trainable": 1926452
825
+ },
826
+ {
827
+ "epoch": 20.784313725490197,
828
+ "grad_norm": 0.9600774645805359,
829
+ "learning_rate": 1.3232096574544602e-05,
830
+ "loss": 3.0228,
831
+ "memory/device_reserved (GiB)": 31.82,
832
+ "memory/max_active (GiB)": 30.03,
833
+ "memory/max_allocated (GiB)": 30.03,
834
+ "ppl": 20.54875,
835
+ "step": 270,
836
+ "tokens/total": 4341760,
837
+ "tokens/train_per_sec_per_gpu": 46.57,
838
+ "tokens/trainable": 1963989
839
+ },
840
+ {
841
+ "epoch": 21.15686274509804,
842
+ "grad_norm": 0.8765286803245544,
843
+ "learning_rate": 1.2995166017866194e-05,
844
+ "loss": 2.9804,
845
+ "memory/device_reserved (GiB)": 31.82,
846
+ "memory/max_active (GiB)": 30.03,
847
+ "memory/max_allocated (GiB)": 30.03,
848
+ "ppl": 19.69569,
849
+ "step": 275,
850
+ "tokens/total": 4419584,
851
+ "tokens/train_per_sec_per_gpu": 46.09,
852
+ "tokens/trainable": 1998647
853
+ },
854
+ {
855
+ "epoch": 21.54901960784314,
856
+ "grad_norm": 0.8645039200782776,
857
+ "learning_rate": 1.2756373558169992e-05,
858
+ "loss": 2.9896,
859
+ "memory/device_reserved (GiB)": 31.82,
860
+ "memory/max_active (GiB)": 30.03,
861
+ "memory/max_allocated (GiB)": 30.03,
862
+ "ppl": 19.87773,
863
+ "step": 280,
864
+ "tokens/total": 4501504,
865
+ "tokens/train_per_sec_per_gpu": 48.25,
866
+ "tokens/trainable": 2036645
867
+ },
868
+ {
869
+ "epoch": 21.941176470588236,
870
+ "grad_norm": 1.0529413223266602,
871
+ "learning_rate": 1.2515867637445088e-05,
872
+ "loss": 3.0176,
873
+ "memory/device_reserved (GiB)": 31.82,
874
+ "memory/max_active (GiB)": 30.03,
875
+ "memory/max_allocated (GiB)": 30.03,
876
+ "ppl": 20.44217,
877
+ "step": 285,
878
+ "tokens/total": 4583424,
879
+ "tokens/train_per_sec_per_gpu": 46.74,
880
+ "tokens/trainable": 2073360
881
+ },
882
+ {
883
+ "epoch": 22.313725490196077,
884
+ "grad_norm": 0.871498167514801,
885
+ "learning_rate": 1.2273797762829615e-05,
886
+ "loss": 2.9589,
887
+ "memory/device_reserved (GiB)": 31.82,
888
+ "memory/max_active (GiB)": 30.03,
889
+ "memory/max_allocated (GiB)": 30.03,
890
+ "ppl": 19.27676,
891
+ "step": 290,
892
+ "tokens/total": 4661248,
893
+ "tokens/train_per_sec_per_gpu": 46.09,
894
+ "tokens/trainable": 2108062
895
+ },
896
+ {
897
+ "epoch": 22.705882352941178,
898
+ "grad_norm": 0.8550048470497131,
899
+ "learning_rate": 1.2030314413671763e-05,
900
+ "loss": 2.9525,
901
+ "memory/device_reserved (GiB)": 31.82,
902
+ "memory/max_active (GiB)": 30.03,
903
+ "memory/max_allocated (GiB)": 30.03,
904
+ "ppl": 19.15378,
905
+ "step": 295,
906
+ "tokens/total": 4743168,
907
+ "tokens/train_per_sec_per_gpu": 50.48,
908
+ "tokens/trainable": 2145259
909
+ },
910
+ {
911
+ "epoch": 23.07843137254902,
912
+ "grad_norm": 0.8834354877471924,
913
+ "learning_rate": 1.1785568947986368e-05,
914
+ "loss": 2.9075,
915
+ "memory/device_reserved (GiB)": 31.82,
916
+ "memory/max_active (GiB)": 30.03,
917
+ "memory/max_allocated (GiB)": 30.03,
918
+ "ppl": 18.31096,
919
+ "step": 300,
920
+ "tokens/total": 4820992,
921
+ "tokens/train_per_sec_per_gpu": 45.14,
922
+ "tokens/trainable": 2180234
923
+ },
924
+ {
925
+ "epoch": 23.07843137254902,
926
+ "eval_loss": 1.8245855569839478,
927
+ "eval_ppl": 6.20022,
928
+ "eval_runtime": 52.3354,
929
+ "eval_samples_per_second": 3.822,
930
+ "eval_steps_per_second": 1.911,
931
+ "memory/device_reserved (GiB)": 31.82,
932
+ "memory/max_active (GiB)": 14.82,
933
+ "memory/max_allocated (GiB)": 14.82,
934
+ "step": 300
935
+ },
936
+ {
937
+ "epoch": 23.470588235294116,
938
+ "grad_norm": 0.9048839211463928,
939
+ "learning_rate": 1.1539713508365336e-05,
940
+ "loss": 2.9687,
941
+ "memory/device_reserved (GiB)": 31.82,
942
+ "memory/max_active (GiB)": 30.03,
943
+ "memory/max_allocated (GiB)": 30.03,
944
+ "ppl": 19.4666,
945
+ "step": 305,
946
+ "tokens/total": 4902912,
947
+ "tokens/train_per_sec_per_gpu": 40.14,
948
+ "tokens/trainable": 2217430
949
+ },
950
+ {
951
+ "epoch": 23.862745098039216,
952
+ "grad_norm": 0.8544889092445374,
953
+ "learning_rate": 1.1292900927400334e-05,
954
+ "loss": 2.8908,
955
+ "memory/device_reserved (GiB)": 31.82,
956
+ "memory/max_active (GiB)": 30.03,
957
+ "memory/max_allocated (GiB)": 30.03,
958
+ "ppl": 18.00771,
959
+ "step": 310,
960
+ "tokens/total": 4984832,
961
+ "tokens/train_per_sec_per_gpu": 44.78,
962
+ "tokens/trainable": 2254924
963
+ },
964
+ {
965
+ "epoch": 24.235294117647058,
966
+ "grad_norm": 0.9118690490722656,
967
+ "learning_rate": 1.1045284632676535e-05,
968
+ "loss": 2.8986,
969
+ "memory/device_reserved (GiB)": 31.82,
970
+ "memory/max_active (GiB)": 30.03,
971
+ "memory/max_allocated (GiB)": 30.03,
972
+ "ppl": 18.14872,
973
+ "step": 315,
974
+ "tokens/total": 5062656,
975
+ "tokens/train_per_sec_per_gpu": 48.84,
976
+ "tokens/trainable": 2289736
977
+ },
978
+ {
979
+ "epoch": 24.627450980392158,
980
+ "grad_norm": 0.9004949927330017,
981
+ "learning_rate": 1.0797018551396527e-05,
982
+ "loss": 2.8844,
983
+ "memory/device_reserved (GiB)": 31.82,
984
+ "memory/max_active (GiB)": 30.03,
985
+ "memory/max_allocated (GiB)": 30.03,
986
+ "ppl": 17.89283,
987
+ "step": 320,
988
+ "tokens/total": 5144576,
989
+ "tokens/train_per_sec_per_gpu": 49.81,
990
+ "tokens/trainable": 2327082
991
+ },
992
+ {
993
+ "epoch": 25.0,
994
+ "grad_norm": 1.1789140701293945,
995
+ "learning_rate": 1.0548257014693602e-05,
996
+ "loss": 2.8413,
997
+ "memory/device_reserved (GiB)": 31.82,
998
+ "memory/max_active (GiB)": 30.03,
999
+ "memory/max_allocated (GiB)": 30.03,
1000
+ "ppl": 17.13803,
1001
+ "step": 325,
1002
+ "tokens/total": 5222400,
1003
+ "tokens/train_per_sec_per_gpu": 44.28,
1004
+ "tokens/trainable": 2361725
1005
+ },
1006
+ {
1007
+ "epoch": 25.392156862745097,
1008
+ "grad_norm": 0.9994716048240662,
1009
+ "learning_rate": 1.0299154661693987e-05,
1010
+ "loss": 2.8495,
1011
+ "memory/device_reserved (GiB)": 31.82,
1012
+ "memory/max_active (GiB)": 30.03,
1013
+ "memory/max_allocated (GiB)": 30.03,
1014
+ "ppl": 17.27914,
1015
+ "step": 330,
1016
+ "tokens/total": 5304320,
1017
+ "tokens/train_per_sec_per_gpu": 44.32,
1018
+ "tokens/trainable": 2398889
1019
+ },
1020
+ {
1021
+ "epoch": 25.784313725490197,
1022
+ "grad_norm": 0.8554360866546631,
1023
+ "learning_rate": 1.0049866343387582e-05,
1024
+ "loss": 2.8331,
1025
+ "memory/device_reserved (GiB)": 31.82,
1026
+ "memory/max_active (GiB)": 30.03,
1027
+ "memory/max_allocated (GiB)": 30.03,
1028
+ "ppl": 16.99807,
1029
+ "step": 335,
1030
+ "tokens/total": 5386240,
1031
+ "tokens/train_per_sec_per_gpu": 49.6,
1032
+ "tokens/trainable": 2436175
1033
+ },
1034
+ {
1035
+ "epoch": 26.15686274509804,
1036
+ "grad_norm": 0.9382306337356567,
1037
+ "learning_rate": 9.800547026367022e-06,
1038
+ "loss": 2.8385,
1039
+ "memory/device_reserved (GiB)": 31.82,
1040
+ "memory/max_active (GiB)": 30.03,
1041
+ "memory/max_allocated (GiB)": 30.03,
1042
+ "ppl": 17.09011,
1043
+ "step": 340,
1044
+ "tokens/total": 5464064,
1045
+ "tokens/train_per_sec_per_gpu": 45.89,
1046
+ "tokens/trainable": 2471548
1047
+ },
1048
+ {
1049
+ "epoch": 26.54901960784314,
1050
+ "grad_norm": 1.0038572549819946,
1051
+ "learning_rate": 9.551351696494854e-06,
1052
+ "loss": 2.8006,
1053
+ "memory/device_reserved (GiB)": 31.82,
1054
+ "memory/max_active (GiB)": 30.03,
1055
+ "memory/max_allocated (GiB)": 30.03,
1056
+ "ppl": 16.45452,
1057
+ "step": 345,
1058
+ "tokens/total": 5545984,
1059
+ "tokens/train_per_sec_per_gpu": 44.95,
1060
+ "tokens/trainable": 2508740
1061
+ },
1062
+ {
1063
+ "epoch": 26.941176470588236,
1064
+ "grad_norm": 0.9205517768859863,
1065
+ "learning_rate": 9.302435262558748e-06,
1066
+ "loss": 2.8267,
1067
+ "memory/device_reserved (GiB)": 31.82,
1068
+ "memory/max_active (GiB)": 30.03,
1069
+ "memory/max_allocated (GiB)": 30.03,
1070
+ "ppl": 16.88963,
1071
+ "step": 350,
1072
+ "tokens/total": 5627904,
1073
+ "tokens/train_per_sec_per_gpu": 45.3,
1074
+ "tokens/trainable": 2545798
1075
+ },
1076
+ {
1077
+ "epoch": 26.941176470588236,
1078
+ "eval_loss": 1.7944940328598022,
1079
+ "eval_ppl": 6.01643,
1080
+ "eval_runtime": 52.4222,
1081
+ "eval_samples_per_second": 3.815,
1082
+ "eval_steps_per_second": 1.908,
1083
+ "memory/device_reserved (GiB)": 31.82,
1084
+ "memory/max_active (GiB)": 14.82,
1085
+ "memory/max_allocated (GiB)": 14.82,
1086
+ "step": 350
1087
+ },
1088
+ {
1089
+ "epoch": 27.313725490196077,
1090
+ "grad_norm": 0.8410999774932861,
1091
+ "learning_rate": 9.05395245997463e-06,
1092
+ "loss": 2.8095,
1093
+ "memory/device_reserved (GiB)": 31.82,
1094
+ "memory/max_active (GiB)": 30.03,
1095
+ "memory/max_allocated (GiB)": 30.03,
1096
+ "ppl": 16.60162,
1097
+ "step": 355,
1098
+ "tokens/total": 5705728,
1099
+ "tokens/train_per_sec_per_gpu": 42.61,
1100
+ "tokens/trainable": 2580317
1101
+ },
1102
+ {
1103
+ "epoch": 27.705882352941178,
1104
+ "grad_norm": 0.9195293188095093,
1105
+ "learning_rate": 8.806057754597559e-06,
1106
+ "loss": 2.8012,
1107
+ "memory/device_reserved (GiB)": 31.82,
1108
+ "memory/max_active (GiB)": 30.03,
1109
+ "memory/max_allocated (GiB)": 30.03,
1110
+ "ppl": 16.46439,
1111
+ "step": 360,
1112
+ "tokens/total": 5787648,
1113
+ "tokens/train_per_sec_per_gpu": 47.59,
1114
+ "tokens/trainable": 2618023
1115
+ },
1116
+ {
1117
+ "epoch": 28.07843137254902,
1118
+ "grad_norm": 1.0124417543411255,
1119
+ "learning_rate": 8.558905246700202e-06,
1120
+ "loss": 2.8255,
1121
+ "memory/device_reserved (GiB)": 31.82,
1122
+ "memory/max_active (GiB)": 30.03,
1123
+ "memory/max_allocated (GiB)": 30.03,
1124
+ "ppl": 16.86938,
1125
+ "step": 365,
1126
+ "tokens/total": 5865472,
1127
+ "tokens/train_per_sec_per_gpu": 46.39,
1128
+ "tokens/trainable": 2652689
1129
+ },
1130
+ {
1131
+ "epoch": 28.470588235294116,
1132
+ "grad_norm": 1.0995986461639404,
1133
+ "learning_rate": 8.312648575178552e-06,
1134
+ "loss": 2.79,
1135
+ "memory/device_reserved (GiB)": 31.82,
1136
+ "memory/max_active (GiB)": 30.03,
1137
+ "memory/max_allocated (GiB)": 30.03,
1138
+ "ppl": 16.28102,
1139
+ "step": 370,
1140
+ "tokens/total": 5947392,
1141
+ "tokens/train_per_sec_per_gpu": 46.65,
1142
+ "tokens/trainable": 2690200
1143
+ },
1144
+ {
1145
+ "epoch": 28.862745098039216,
1146
+ "grad_norm": 0.9146213531494141,
1147
+ "learning_rate": 8.06744082204447e-06,
1148
+ "loss": 2.7736,
1149
+ "memory/device_reserved (GiB)": 31.82,
1150
+ "memory/max_active (GiB)": 30.03,
1151
+ "memory/max_allocated (GiB)": 30.03,
1152
+ "ppl": 16.01619,
1153
+ "step": 375,
1154
+ "tokens/total": 6029312,
1155
+ "tokens/train_per_sec_per_gpu": 44.2,
1156
+ "tokens/trainable": 2727130
1157
+ },
1158
+ {
1159
+ "epoch": 29.235294117647058,
1160
+ "grad_norm": 1.1337180137634277,
1161
+ "learning_rate": 7.823434417264378e-06,
1162
+ "loss": 2.7499,
1163
+ "memory/device_reserved (GiB)": 31.82,
1164
+ "memory/max_active (GiB)": 30.03,
1165
+ "memory/max_allocated (GiB)": 30.03,
1166
+ "ppl": 15.64107,
1167
+ "step": 380,
1168
+ "tokens/total": 6107136,
1169
+ "tokens/train_per_sec_per_gpu": 44.25,
1170
+ "tokens/trainable": 2761625
1171
+ },
1172
+ {
1173
+ "epoch": 29.627450980392158,
1174
+ "grad_norm": 0.8825079202651978,
1175
+ "learning_rate": 7.580781044003324e-06,
1176
+ "loss": 2.7665,
1177
+ "memory/device_reserved (GiB)": 31.82,
1178
+ "memory/max_active (GiB)": 30.03,
1179
+ "memory/max_allocated (GiB)": 30.03,
1180
+ "ppl": 15.90288,
1181
+ "step": 385,
1182
+ "tokens/total": 6189056,
1183
+ "tokens/train_per_sec_per_gpu": 44.98,
1184
+ "tokens/trainable": 2799048
1185
+ },
1186
+ {
1187
+ "epoch": 30.0,
1188
+ "grad_norm": 1.2486763000488281,
1189
+ "learning_rate": 7.33963154433325e-06,
1190
+ "loss": 2.7715,
1191
+ "memory/device_reserved (GiB)": 31.82,
1192
+ "memory/max_active (GiB)": 30.03,
1193
+ "memory/max_allocated (GiB)": 30.03,
1194
+ "ppl": 15.98259,
1195
+ "step": 390,
1196
+ "tokens/total": 6266880,
1197
+ "tokens/train_per_sec_per_gpu": 44.08,
1198
+ "tokens/trainable": 2834070
1199
+ },
1200
+ {
1201
+ "epoch": 30.392156862745097,
1202
+ "grad_norm": 1.1907079219818115,
1203
+ "learning_rate": 7.100135825464138e-06,
1204
+ "loss": 2.7558,
1205
+ "memory/device_reserved (GiB)": 31.82,
1206
+ "memory/max_active (GiB)": 30.03,
1207
+ "memory/max_allocated (GiB)": 30.03,
1208
+ "ppl": 15.73362,
1209
+ "step": 395,
1210
+ "tokens/total": 6348800,
1211
+ "tokens/train_per_sec_per_gpu": 48.43,
1212
+ "tokens/trainable": 2871536
1213
+ },
1214
+ {
1215
+ "epoch": 30.784313725490197,
1216
+ "grad_norm": 0.8881621360778809,
1217
+ "learning_rate": 6.862442766556297e-06,
1218
+ "loss": 2.7239,
1219
+ "memory/device_reserved (GiB)": 31.82,
1220
+ "memory/max_active (GiB)": 30.03,
1221
+ "memory/max_allocated (GiB)": 30.03,
1222
+ "ppl": 15.23964,
1223
+ "step": 400,
1224
+ "tokens/total": 6430720,
1225
+ "tokens/train_per_sec_per_gpu": 43.32,
1226
+ "tokens/trainable": 2909069
1227
+ },
1228
+ {
1229
+ "epoch": 30.784313725490197,
1230
+ "eval_loss": 1.7793785333633423,
1231
+ "eval_ppl": 5.92617,
1232
+ "eval_runtime": 52.3688,
1233
+ "eval_samples_per_second": 3.819,
1234
+ "eval_steps_per_second": 1.91,
1235
+ "memory/device_reserved (GiB)": 31.82,
1236
+ "memory/max_active (GiB)": 14.82,
1237
+ "memory/max_allocated (GiB)": 14.82,
1238
+ "step": 400
1239
+ },
1240
+ {
1241
+ "epoch": 31.15686274509804,
1242
+ "grad_norm": 0.8862964510917664,
1243
+ "learning_rate": 6.6267001261717015e-06,
1244
+ "loss": 2.741,
1245
+ "memory/device_reserved (GiB)": 31.82,
1246
+ "memory/max_active (GiB)": 30.03,
1247
+ "memory/max_allocated (GiB)": 30.03,
1248
+ "ppl": 15.50248,
1249
+ "step": 405,
1250
+ "tokens/total": 6508544,
1251
+ "tokens/train_per_sec_per_gpu": 46.64,
1252
+ "tokens/trainable": 2943351
1253
+ },
1254
+ {
1255
+ "epoch": 31.54901960784314,
1256
+ "grad_norm": 0.9779984354972839,
1257
+ "learning_rate": 6.393054450421963e-06,
1258
+ "loss": 2.7685,
1259
+ "memory/device_reserved (GiB)": 31.82,
1260
+ "memory/max_active (GiB)": 30.03,
1261
+ "memory/max_allocated (GiB)": 30.03,
1262
+ "ppl": 15.93471,
1263
+ "step": 410,
1264
+ "tokens/total": 6590464,
1265
+ "tokens/train_per_sec_per_gpu": 46.13,
1266
+ "tokens/trainable": 2981163
1267
+ },
1268
+ {
1269
+ "epoch": 31.941176470588236,
1270
+ "grad_norm": 1.168878197669983,
1271
+ "learning_rate": 6.1616509818699975e-06,
1272
+ "loss": 2.7398,
1273
+ "memory/device_reserved (GiB)": 31.82,
1274
+ "memory/max_active (GiB)": 30.03,
1275
+ "memory/max_allocated (GiB)": 30.03,
1276
+ "ppl": 15.48389,
1277
+ "step": 415,
1278
+ "tokens/total": 6672384,
1279
+ "tokens/train_per_sec_per_gpu": 45.31,
1280
+ "tokens/trainable": 3017999
1281
+ },
1282
+ {
1283
+ "epoch": 32.31372549019608,
1284
+ "grad_norm": 1.2255523204803467,
1285
+ "learning_rate": 5.932633569242e-06,
1286
+ "loss": 2.7219,
1287
+ "memory/device_reserved (GiB)": 31.82,
1288
+ "memory/max_active (GiB)": 30.03,
1289
+ "memory/max_allocated (GiB)": 30.03,
1290
+ "ppl": 15.20919,
1291
+ "step": 420,
1292
+ "tokens/total": 6750208,
1293
+ "tokens/train_per_sec_per_gpu": 47.33,
1294
+ "tokens/trainable": 3052861
1295
+ },
1296
+ {
1297
+ "epoch": 32.705882352941174,
1298
+ "grad_norm": 0.891295850276947,
1299
+ "learning_rate": 5.706144578005908e-06,
1300
+ "loss": 2.7354,
1301
+ "memory/device_reserved (GiB)": 31.82,
1302
+ "memory/max_active (GiB)": 30.03,
1303
+ "memory/max_allocated (GiB)": 30.03,
1304
+ "ppl": 15.41591,
1305
+ "step": 425,
1306
+ "tokens/total": 6832128,
1307
+ "tokens/train_per_sec_per_gpu": 46.14,
1308
+ "tokens/trainable": 3090169
1309
+ },
1310
+ {
1311
+ "epoch": 33.07843137254902,
1312
+ "grad_norm": 0.8626319766044617,
1313
+ "learning_rate": 5.4823248018719184e-06,
1314
+ "loss": 2.6909,
1315
+ "memory/device_reserved (GiB)": 31.82,
1316
+ "memory/max_active (GiB)": 30.03,
1317
+ "memory/max_allocated (GiB)": 30.03,
1318
+ "ppl": 14.74494,
1319
+ "step": 430,
1320
+ "tokens/total": 6909952,
1321
+ "tokens/train_per_sec_per_gpu": 43.16,
1322
+ "tokens/trainable": 3124926
1323
+ },
1324
+ {
1325
+ "epoch": 33.470588235294116,
1326
+ "grad_norm": 1.3728200197219849,
1327
+ "learning_rate": 5.2613133752700145e-06,
1328
+ "loss": 2.7291,
1329
+ "memory/device_reserved (GiB)": 31.82,
1330
+ "memory/max_active (GiB)": 30.03,
1331
+ "memory/max_allocated (GiB)": 30.03,
1332
+ "ppl": 15.31909,
1333
+ "step": 435,
1334
+ "tokens/total": 6991872,
1335
+ "tokens/train_per_sec_per_gpu": 44.19,
1336
+ "tokens/trainable": 3162036
1337
+ },
1338
+ {
1339
+ "epoch": 33.86274509803921,
1340
+ "grad_norm": 1.0412100553512573,
1341
+ "learning_rate": 5.043247686859024e-06,
1342
+ "loss": 2.7137,
1343
+ "memory/device_reserved (GiB)": 31.82,
1344
+ "memory/max_active (GiB)": 30.03,
1345
+ "memory/max_allocated (GiB)": 30.03,
1346
+ "ppl": 15.08499,
1347
+ "step": 440,
1348
+ "tokens/total": 7073792,
1349
+ "tokens/train_per_sec_per_gpu": 49.59,
1350
+ "tokens/trainable": 3199398
1351
+ },
1352
+ {
1353
+ "epoch": 34.23529411764706,
1354
+ "grad_norm": 1.0112348794937134,
1355
+ "learning_rate": 4.8282632941208725e-06,
1356
+ "loss": 2.6506,
1357
+ "memory/device_reserved (GiB)": 31.82,
1358
+ "memory/max_active (GiB)": 30.03,
1359
+ "memory/max_allocated (GiB)": 30.03,
1360
+ "ppl": 14.16253,
1361
+ "step": 445,
1362
+ "tokens/total": 7151616,
1363
+ "tokens/train_per_sec_per_gpu": 45.39,
1364
+ "tokens/trainable": 3234014
1365
+ },
1366
+ {
1367
+ "epoch": 34.627450980392155,
1368
+ "grad_norm": 0.9728626608848572,
1369
+ "learning_rate": 4.616493839093179e-06,
1370
+ "loss": 2.7275,
1371
+ "memory/device_reserved (GiB)": 31.82,
1372
+ "memory/max_active (GiB)": 30.03,
1373
+ "memory/max_allocated (GiB)": 30.03,
1374
+ "ppl": 15.2946,
1375
+ "step": 450,
1376
+ "tokens/total": 7233536,
1377
+ "tokens/train_per_sec_per_gpu": 43.8,
1378
+ "tokens/trainable": 3271273
1379
+ },
1380
+ {
1381
+ "epoch": 34.627450980392155,
1382
+ "eval_loss": 1.7696921825408936,
1383
+ "eval_ppl": 5.86905,
1384
+ "eval_runtime": 52.5443,
1385
+ "eval_samples_per_second": 3.806,
1386
+ "eval_steps_per_second": 1.903,
1387
+ "memory/device_reserved (GiB)": 31.82,
1388
+ "memory/max_active (GiB)": 14.82,
1389
+ "memory/max_allocated (GiB)": 14.82,
1390
+ "step": 450
1391
+ },
1392
+ {
1393
+ "epoch": 35.0,
1394
+ "grad_norm": 1.2022005319595337,
1395
+ "learning_rate": 4.408070965292534e-06,
1396
+ "loss": 2.6792,
1397
+ "memory/device_reserved (GiB)": 31.82,
1398
+ "memory/max_active (GiB)": 30.03,
1399
+ "memory/max_allocated (GiB)": 30.03,
1400
+ "ppl": 14.57343,
1401
+ "step": 455,
1402
+ "tokens/total": 7311360,
1403
+ "tokens/train_per_sec_per_gpu": 44.61,
1404
+ "tokens/trainable": 3306415
1405
+ },
1406
+ {
1407
+ "epoch": 35.3921568627451,
1408
+ "grad_norm": 0.9974932074546814,
1409
+ "learning_rate": 4.203124235880179e-06,
1410
+ "loss": 2.6945,
1411
+ "memory/device_reserved (GiB)": 31.82,
1412
+ "memory/max_active (GiB)": 30.03,
1413
+ "memory/max_allocated (GiB)": 30.03,
1414
+ "ppl": 14.79812,
1415
+ "step": 460,
1416
+ "tokens/total": 7393280,
1417
+ "tokens/train_per_sec_per_gpu": 49.35,
1418
+ "tokens/trainable": 3343607
1419
+ },
1420
+ {
1421
+ "epoch": 35.78431372549019,
1422
+ "grad_norm": 0.9656805992126465,
1423
+ "learning_rate": 4.001781053120863e-06,
1424
+ "loss": 2.6776,
1425
+ "memory/device_reserved (GiB)": 31.82,
1426
+ "memory/max_active (GiB)": 30.03,
1427
+ "memory/max_allocated (GiB)": 30.03,
1428
+ "ppl": 14.55013,
1429
+ "step": 465,
1430
+ "tokens/total": 7475200,
1431
+ "tokens/train_per_sec_per_gpu": 43.58,
1432
+ "tokens/trainable": 3380809
1433
+ },
1434
+ {
1435
+ "epoch": 36.15686274509804,
1436
+ "grad_norm": 0.885778546333313,
1437
+ "learning_rate": 3.804166579185018e-06,
1438
+ "loss": 2.729,
1439
+ "memory/device_reserved (GiB)": 31.82,
1440
+ "memory/max_active (GiB)": 30.03,
1441
+ "memory/max_allocated (GiB)": 30.03,
1442
+ "ppl": 15.31756,
1443
+ "step": 470,
1444
+ "tokens/total": 7553024,
1445
+ "tokens/train_per_sec_per_gpu": 45.29,
1446
+ "tokens/trainable": 3415795
1447
+ },
1448
+ {
1449
+ "epoch": 36.549019607843135,
1450
+ "grad_norm": 1.0801104307174683,
1451
+ "learning_rate": 3.610403658343443e-06,
1452
+ "loss": 2.6498,
1453
+ "memory/device_reserved (GiB)": 31.82,
1454
+ "memory/max_active (GiB)": 30.03,
1455
+ "memory/max_allocated (GiB)": 30.03,
1456
+ "ppl": 14.15121,
1457
+ "step": 475,
1458
+ "tokens/total": 7634944,
1459
+ "tokens/train_per_sec_per_gpu": 45.39,
1460
+ "tokens/trainable": 3453063
1461
+ },
1462
+ {
1463
+ "epoch": 36.94117647058823,
1464
+ "grad_norm": 1.0326086282730103,
1465
+ "learning_rate": 3.4206127406028744e-06,
1466
+ "loss": 2.702,
1467
+ "memory/device_reserved (GiB)": 31.82,
1468
+ "memory/max_active (GiB)": 30.03,
1469
+ "memory/max_allocated (GiB)": 30.03,
1470
+ "ppl": 14.90952,
1471
+ "step": 480,
1472
+ "tokens/total": 7716864,
1473
+ "tokens/train_per_sec_per_gpu": 43.99,
1474
+ "tokens/trainable": 3490318
1475
+ },
1476
+ {
1477
+ "epoch": 37.31372549019608,
1478
+ "grad_norm": 1.074462652206421,
1479
+ "learning_rate": 3.234911806829948e-06,
1480
+ "loss": 2.6897,
1481
+ "memory/device_reserved (GiB)": 31.82,
1482
+ "memory/max_active (GiB)": 30.03,
1483
+ "memory/max_allocated (GiB)": 30.03,
1484
+ "ppl": 14.72726,
1485
+ "step": 485,
1486
+ "tokens/total": 7794688,
1487
+ "tokens/train_per_sec_per_gpu": 44.84,
1488
+ "tokens/trainable": 3524897
1489
+ },
1490
+ {
1491
+ "epoch": 37.705882352941174,
1492
+ "grad_norm": 0.9462416768074036,
1493
+ "learning_rate": 3.0534162954100264e-06,
1494
+ "loss": 2.7005,
1495
+ "memory/device_reserved (GiB)": 31.82,
1496
+ "memory/max_active (GiB)": 30.03,
1497
+ "memory/max_allocated (GiB)": 30.03,
1498
+ "ppl": 14.88717,
1499
+ "step": 490,
1500
+ "tokens/total": 7876608,
1501
+ "tokens/train_per_sec_per_gpu": 46.96,
1502
+ "tokens/trainable": 3562370
1503
+ },
1504
+ {
1505
+ "epoch": 38.07843137254902,
1506
+ "grad_norm": 1.0124855041503906,
1507
+ "learning_rate": 2.876239030486554e-06,
1508
+ "loss": 2.65,
1509
+ "memory/device_reserved (GiB)": 31.82,
1510
+ "memory/max_active (GiB)": 30.03,
1511
+ "memory/max_allocated (GiB)": 30.03,
1512
+ "ppl": 14.15404,
1513
+ "step": 495,
1514
+ "tokens/total": 7954432,
1515
+ "tokens/train_per_sec_per_gpu": 43.07,
1516
+ "tokens/trainable": 3596824
1517
+ },
1518
+ {
1519
+ "epoch": 38.470588235294116,
1520
+ "grad_norm": 1.0789132118225098,
1521
+ "learning_rate": 2.703490151825492e-06,
1522
+ "loss": 2.6912,
1523
+ "memory/device_reserved (GiB)": 31.82,
1524
+ "memory/max_active (GiB)": 30.03,
1525
+ "memory/max_allocated (GiB)": 30.03,
1526
+ "ppl": 14.74936,
1527
+ "step": 500,
1528
+ "tokens/total": 8036352,
1529
+ "tokens/train_per_sec_per_gpu": 46.11,
1530
+ "tokens/trainable": 3634339
1531
+ },
1532
+ {
1533
+ "epoch": 38.470588235294116,
1534
+ "eval_loss": 1.763442039489746,
1535
+ "eval_ppl": 5.83248,
1536
+ "eval_runtime": 52.5284,
1537
+ "eval_samples_per_second": 3.807,
1538
+ "eval_steps_per_second": 1.904,
1539
+ "memory/device_reserved (GiB)": 31.82,
1540
+ "memory/max_active (GiB)": 14.82,
1541
+ "memory/max_allocated (GiB)": 14.82,
1542
+ "step": 500
1543
+ }
1544
+ ],
1545
+ "logging_steps": 5,
1546
+ "max_steps": 650,
1547
+ "num_input_tokens_seen": 0,
1548
+ "num_train_epochs": 50,
1549
+ "save_steps": 100,
1550
+ "stateful_callbacks": {
1551
+ "TrainerControl": {
1552
+ "args": {
1553
+ "should_epoch_stop": false,
1554
+ "should_evaluate": false,
1555
+ "should_log": false,
1556
+ "should_save": true,
1557
+ "should_training_stop": false
1558
+ },
1559
+ "attributes": {}
1560
+ }
1561
+ },
1562
+ "total_flos": 1.5540687882472653e+17,
1563
+ "train_batch_size": 2,
1564
+ "trial_name": null,
1565
+ "trial_params": null
1566
+ }
checkpoint-500/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a94bb930d8dd65dcaaaff77c43d92a52c500cdd1a9193a036a0d8e7038b4e8d1
3
+ size 9425
checkpoint-500/vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-500/zero_to_fp32.py ADDED
@@ -0,0 +1,760 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) Microsoft Corporation.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # DeepSpeed Team
7
+
8
+ # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
+ # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
+ # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
+ # application.
12
+ #
13
+ # example:
14
+ # python zero_to_fp32.py . output_dir/
15
+ # or
16
+ # python zero_to_fp32.py . output_dir/ --safe_serialization
17
+
18
+ import argparse
19
+ import torch
20
+ import glob
21
+ import math
22
+ import os
23
+ import re
24
+ import gc
25
+ import json
26
+ import numpy as np
27
+ from tqdm import tqdm
28
+ from collections import OrderedDict
29
+ from dataclasses import dataclass
30
+
31
+ # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
32
+ # DeepSpeed data structures it has to be available in the current python environment.
33
+ from deepspeed.utils import logger
34
+ from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
35
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
36
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
37
+
38
+
39
+ @dataclass
40
+ class zero_model_state:
41
+ buffers: dict()
42
+ param_shapes: dict()
43
+ shared_params: list
44
+ ds_version: int
45
+ frozen_param_shapes: dict()
46
+ frozen_param_fragments: dict()
47
+
48
+
49
+ debug = 0
50
+
51
+ # load to cpu
52
+ device = torch.device('cpu')
53
+
54
+
55
+ def atoi(text):
56
+ return int(text) if text.isdigit() else text
57
+
58
+
59
+ def natural_keys(text):
60
+ '''
61
+ alist.sort(key=natural_keys) sorts in human order
62
+ http://nedbatchelder.com/blog/200712/human_sorting.html
63
+ (See Toothy's implementation in the comments)
64
+ '''
65
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
66
+
67
+
68
+ def get_model_state_file(checkpoint_dir, zero_stage):
69
+ if not os.path.isdir(checkpoint_dir):
70
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
71
+
72
+ # there should be only one file
73
+ if zero_stage <= 2:
74
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
75
+ elif zero_stage == 3:
76
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
77
+
78
+ if not os.path.exists(file):
79
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
80
+
81
+ return file
82
+
83
+
84
+ def get_checkpoint_files(checkpoint_dir, glob_pattern):
85
+ # XXX: need to test that this simple glob rule works for multi-node setup too
86
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
87
+
88
+ if len(ckpt_files) == 0:
89
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
90
+
91
+ return ckpt_files
92
+
93
+
94
+ def get_optim_files(checkpoint_dir):
95
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
96
+
97
+
98
+ def get_model_state_files(checkpoint_dir):
99
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
100
+
101
+
102
+ def parse_model_states(files):
103
+ zero_model_states = []
104
+ for file in files:
105
+ state_dict = torch.load(file, map_location=device, weights_only=False)
106
+
107
+ if BUFFER_NAMES not in state_dict:
108
+ raise ValueError(f"{file} is not a model state checkpoint")
109
+ buffer_names = state_dict[BUFFER_NAMES]
110
+ if debug:
111
+ print("Found buffers:", buffer_names)
112
+
113
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
114
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
115
+ param_shapes = state_dict[PARAM_SHAPES]
116
+
117
+ # collect parameters that are included in param_shapes
118
+ param_names = []
119
+ for s in param_shapes:
120
+ for name in s.keys():
121
+ param_names.append(name)
122
+
123
+ # update with frozen parameters
124
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
125
+ if frozen_param_shapes is not None:
126
+ if debug:
127
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
128
+ param_names += list(frozen_param_shapes.keys())
129
+
130
+ # handle shared params
131
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
132
+
133
+ ds_version = state_dict.get(DS_VERSION, None)
134
+
135
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
136
+
137
+ z_model_state = zero_model_state(buffers=buffers,
138
+ param_shapes=param_shapes,
139
+ shared_params=shared_params,
140
+ ds_version=ds_version,
141
+ frozen_param_shapes=frozen_param_shapes,
142
+ frozen_param_fragments=frozen_param_fragments)
143
+ zero_model_states.append(z_model_state)
144
+
145
+ return zero_model_states
146
+
147
+
148
+ def parse_optim_states(files, ds_checkpoint_dir):
149
+ total_files = len(files)
150
+ state_dicts = []
151
+ for f in tqdm(files, desc='Loading checkpoint shards'):
152
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
153
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
154
+ # and also handle the case where it was already removed by another helper script
155
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
156
+ state_dicts.append(state_dict)
157
+
158
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
159
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
160
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
161
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
162
+
163
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
164
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
165
+ # use the max of the partition_count to get the dp world_size.
166
+
167
+ if type(world_size) is list:
168
+ world_size = max(world_size)
169
+
170
+ if world_size != total_files:
171
+ raise ValueError(
172
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
173
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
174
+ )
175
+
176
+ # the groups are named differently in each stage
177
+ if zero_stage <= 2:
178
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
179
+ elif zero_stage == 3:
180
+ fp32_groups_key = FP32_FLAT_GROUPS
181
+ else:
182
+ raise ValueError(f"unknown zero stage {zero_stage}")
183
+
184
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
185
+ return zero_stage, world_size, fp32_flat_groups
186
+
187
+
188
+ def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
189
+ """
190
+ Returns fp32 state_dict reconstructed from ds checkpoint
191
+
192
+ Args:
193
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
194
+
195
+ """
196
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
197
+
198
+ optim_files = get_optim_files(ds_checkpoint_dir)
199
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
200
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
201
+
202
+ model_files = get_model_state_files(ds_checkpoint_dir)
203
+
204
+ zero_model_states = parse_model_states(model_files)
205
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
206
+
207
+ if zero_stage <= 2:
208
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
209
+ exclude_frozen_parameters)
210
+ elif zero_stage == 3:
211
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
212
+ exclude_frozen_parameters)
213
+
214
+
215
+ def _zero2_merge_frozen_params(state_dict, zero_model_states):
216
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
217
+ return
218
+
219
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
220
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
221
+
222
+ if debug:
223
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
224
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
225
+
226
+ wanted_params = len(frozen_param_shapes)
227
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
228
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
229
+ print(f'Frozen params: Have {avail_numel} numels to process.')
230
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
231
+
232
+ total_params = 0
233
+ total_numel = 0
234
+ for name, shape in frozen_param_shapes.items():
235
+ total_params += 1
236
+ unpartitioned_numel = shape.numel()
237
+ total_numel += unpartitioned_numel
238
+
239
+ state_dict[name] = frozen_param_fragments[name]
240
+
241
+ if debug:
242
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
243
+
244
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
245
+
246
+
247
+ def _has_callable(obj, fn):
248
+ attr = getattr(obj, fn, None)
249
+ return callable(attr)
250
+
251
+
252
+ def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
253
+ param_shapes = zero_model_states[0].param_shapes
254
+
255
+ # Reconstruction protocol:
256
+ #
257
+ # XXX: document this
258
+
259
+ if debug:
260
+ for i in range(world_size):
261
+ for j in range(len(fp32_flat_groups[0])):
262
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
263
+
264
+ # XXX: memory usage doubles here (zero2)
265
+ num_param_groups = len(fp32_flat_groups[0])
266
+ merged_single_partition_of_fp32_groups = []
267
+ for i in range(num_param_groups):
268
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
269
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
270
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
271
+ avail_numel = sum(
272
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
273
+
274
+ if debug:
275
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
276
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
277
+ # not asserting if there is a mismatch due to possible padding
278
+ print(f"Have {avail_numel} numels to process.")
279
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
280
+
281
+ # params
282
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
283
+ # out-of-core computing solution
284
+ total_numel = 0
285
+ total_params = 0
286
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
287
+ offset = 0
288
+ avail_numel = full_single_fp32_vector.numel()
289
+ for name, shape in shapes.items():
290
+
291
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
292
+ total_numel += unpartitioned_numel
293
+ total_params += 1
294
+
295
+ if debug:
296
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
297
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
298
+ offset += unpartitioned_numel
299
+
300
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
301
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
302
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
303
+ # live optimizer object, so we are checking that the numbers are within the right range
304
+ align_to = 2 * world_size
305
+
306
+ def zero2_align(x):
307
+ return align_to * math.ceil(x / align_to)
308
+
309
+ if debug:
310
+ print(f"original offset={offset}, avail_numel={avail_numel}")
311
+
312
+ offset = zero2_align(offset)
313
+ avail_numel = zero2_align(avail_numel)
314
+
315
+ if debug:
316
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
317
+
318
+ # Sanity check
319
+ if offset != avail_numel:
320
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
321
+
322
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
323
+
324
+
325
+ def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
326
+ exclude_frozen_parameters):
327
+ state_dict = OrderedDict()
328
+
329
+ # buffers
330
+ buffers = zero_model_states[0].buffers
331
+ state_dict.update(buffers)
332
+ if debug:
333
+ print(f"added {len(buffers)} buffers")
334
+
335
+ if not exclude_frozen_parameters:
336
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
337
+
338
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
339
+
340
+ # recover shared parameters
341
+ for pair in zero_model_states[0].shared_params:
342
+ if pair[1] in state_dict:
343
+ state_dict[pair[0]] = state_dict[pair[1]]
344
+
345
+ return state_dict
346
+
347
+
348
+ def zero3_partitioned_param_info(unpartitioned_numel, world_size):
349
+ remainder = unpartitioned_numel % world_size
350
+ padding_numel = (world_size - remainder) if remainder else 0
351
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
352
+ return partitioned_numel, padding_numel
353
+
354
+
355
+ def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
356
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
357
+ return
358
+
359
+ if debug:
360
+ for i in range(world_size):
361
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
362
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
363
+
364
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
365
+ wanted_params = len(frozen_param_shapes)
366
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
367
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
368
+ print(f'Frozen params: Have {avail_numel} numels to process.')
369
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
370
+
371
+ total_params = 0
372
+ total_numel = 0
373
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
374
+ total_params += 1
375
+ unpartitioned_numel = shape.numel()
376
+ total_numel += unpartitioned_numel
377
+
378
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
379
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
380
+
381
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
382
+
383
+ if debug:
384
+ print(
385
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
386
+ )
387
+
388
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
389
+
390
+
391
+ class GatheredTensor:
392
+ """
393
+ A pseudo tensor that collects partitioned weights.
394
+ It is more memory efficient when there are multiple groups.
395
+ """
396
+
397
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
398
+ self.flat_groups = flat_groups
399
+ self.flat_groups_offset = flat_groups_offset
400
+ self.offset = offset
401
+ self.partitioned_numel = partitioned_numel
402
+ self.shape = shape
403
+ self.dtype = self.flat_groups[0][0].dtype
404
+
405
+ def contiguous(self):
406
+ """
407
+ Merge partitioned weights from flat_groups into a single tensor.
408
+ """
409
+ end_idx = self.offset + self.partitioned_numel
410
+ world_size = len(self.flat_groups)
411
+ pad_flat_param_chunks = []
412
+
413
+ for rank_i in range(world_size):
414
+ # for each rank, we need to collect weights from related group/groups
415
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
416
+ start_group_id = None
417
+ end_group_id = None
418
+ for group_id in range(len(self.flat_groups_offset)):
419
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
420
+ start_group_id = group_id
421
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
422
+ end_group_id = group_id
423
+ break
424
+ # collect weights from related group/groups
425
+ for group_id in range(start_group_id, end_group_id + 1):
426
+ flat_tensor = flat_groups_at_rank_i[group_id]
427
+ start_offset = self.offset - self.flat_groups_offset[group_id]
428
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
429
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
430
+
431
+ # collect weights from all ranks
432
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
433
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
434
+ return param
435
+
436
+
437
+ def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
438
+ param_shapes = zero_model_states[0].param_shapes
439
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
440
+
441
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
442
+ # param, re-consolidating each param, while dealing with padding if any
443
+
444
+ # merge list of dicts, preserving order
445
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
446
+
447
+ if debug:
448
+ for i in range(world_size):
449
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
450
+
451
+ wanted_params = len(param_shapes)
452
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
453
+ # not asserting if there is a mismatch due to possible padding
454
+ avail_numel = fp32_flat_groups[0].numel() * world_size
455
+ print(f"Trainable params: Have {avail_numel} numels to process.")
456
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
457
+
458
+ # params
459
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
460
+ # out-of-core computing solution
461
+ offset = 0
462
+ total_numel = 0
463
+ total_params = 0
464
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
465
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
466
+ unpartitioned_numel = shape.numel()
467
+ total_numel += unpartitioned_numel
468
+ total_params += 1
469
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
470
+
471
+ if debug:
472
+ print(
473
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
474
+ )
475
+
476
+ # memory efficient tensor
477
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
478
+ state_dict[name] = tensor
479
+ offset += partitioned_numel
480
+
481
+ offset *= world_size
482
+
483
+ # Sanity check
484
+ if offset != avail_numel:
485
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
486
+
487
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
488
+
489
+
490
+ def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
491
+ exclude_frozen_parameters):
492
+ state_dict = OrderedDict()
493
+
494
+ # buffers
495
+ buffers = zero_model_states[0].buffers
496
+ state_dict.update(buffers)
497
+ if debug:
498
+ print(f"added {len(buffers)} buffers")
499
+
500
+ if not exclude_frozen_parameters:
501
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
502
+
503
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
504
+
505
+ # recover shared parameters
506
+ for pair in zero_model_states[0].shared_params:
507
+ if pair[1] in state_dict:
508
+ state_dict[pair[0]] = state_dict[pair[1]]
509
+
510
+ return state_dict
511
+
512
+
513
+ def to_torch_tensor(state_dict, return_empty_tensor=False):
514
+ """
515
+ Convert state_dict of GatheredTensor to torch tensor
516
+ """
517
+ torch_state_dict = {}
518
+ converted_tensors = {}
519
+ for name, tensor in state_dict.items():
520
+ tensor_id = id(tensor)
521
+ if tensor_id in converted_tensors: # shared tensors
522
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
523
+ torch_state_dict[name] = shared_tensor
524
+ else:
525
+ converted_tensors[tensor_id] = name
526
+ if return_empty_tensor:
527
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
528
+ else:
529
+ torch_state_dict[name] = tensor.contiguous()
530
+ return torch_state_dict
531
+
532
+
533
+ def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
534
+ tag=None,
535
+ exclude_frozen_parameters=False,
536
+ lazy_mode=False):
537
+ """
538
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
539
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
540
+ via a model hub.
541
+
542
+ Args:
543
+ - ``checkpoint_dir``: path to the desired checkpoint folder
544
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
545
+ - ``exclude_frozen_parameters``: exclude frozen parameters
546
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
547
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
548
+
549
+ Returns:
550
+ - pytorch ``state_dict``
551
+
552
+ A typical usage might be ::
553
+
554
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
555
+ # do the training and checkpoint saving
556
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
557
+ model = model.cpu() # move to cpu
558
+ model.load_state_dict(state_dict)
559
+ # submit to model hub or save the model to share with others
560
+
561
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
562
+ application. i.e. you will need to re-initialize the deepspeed engine, since
563
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
564
+
565
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
566
+
567
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
568
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
569
+ the checkpoint. Or you can load state_dict in lazy mode ::
570
+
571
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
572
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
573
+ for name, lazy_tensor in state_dict.item():
574
+ tensor = lazy_tensor.contiguous() # to cpu
575
+ print(name, tensor)
576
+ # del tensor to release memory if it no longer in use
577
+ """
578
+ if tag is None:
579
+ latest_path = os.path.join(checkpoint_dir, 'latest')
580
+ if os.path.isfile(latest_path):
581
+ with open(latest_path, 'r') as fd:
582
+ tag = fd.read().strip()
583
+ else:
584
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
585
+
586
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
587
+
588
+ if not os.path.isdir(ds_checkpoint_dir):
589
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
590
+
591
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
592
+ if lazy_mode:
593
+ return state_dict
594
+ else:
595
+ return to_torch_tensor(state_dict)
596
+
597
+
598
+ def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
599
+ output_dir,
600
+ max_shard_size="5GB",
601
+ safe_serialization=False,
602
+ tag=None,
603
+ exclude_frozen_parameters=False):
604
+ """
605
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
606
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
607
+
608
+ Args:
609
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
610
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
611
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
612
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
613
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
614
+ - ``exclude_frozen_parameters``: exclude frozen parameters
615
+ """
616
+
617
+ # Dependency pre-check
618
+ if safe_serialization:
619
+ try:
620
+ from safetensors.torch import save_file
621
+ except ImportError:
622
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
623
+ raise
624
+ if max_shard_size is not None:
625
+ try:
626
+ from huggingface_hub import split_torch_state_dict_into_shards
627
+ except ImportError:
628
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
629
+ raise
630
+
631
+ # Convert zero checkpoint to state_dict
632
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
633
+ tag,
634
+ exclude_frozen_parameters,
635
+ lazy_mode=True)
636
+
637
+ # Shard the model if it is too big.
638
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
639
+ if max_shard_size is not None:
640
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
641
+ # an memory-efficient approach for sharding
642
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
643
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
644
+ filename_pattern=filename_pattern,
645
+ max_shard_size=max_shard_size)
646
+ else:
647
+ from collections import namedtuple
648
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
649
+ state_dict_split = StateDictSplit(is_sharded=False,
650
+ filename_to_tensors={weights_name: list(state_dict.keys())})
651
+
652
+ # Save the model by shard
653
+ os.makedirs(output_dir, exist_ok=True)
654
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
655
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
656
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
657
+ shard_state_dict = to_torch_tensor(shard_state_dict)
658
+ output_path = os.path.join(output_dir, shard_file)
659
+ if safe_serialization:
660
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
661
+ else:
662
+ torch.save(shard_state_dict, output_path)
663
+ # release the memory of current shard
664
+ for tensor_name in list(shard_state_dict.keys()):
665
+ del state_dict[tensor_name]
666
+ del shard_state_dict[tensor_name]
667
+ del shard_state_dict
668
+ gc.collect()
669
+
670
+ # Save index if sharded
671
+ if state_dict_split.is_sharded:
672
+ index = {
673
+ "metadata": state_dict_split.metadata,
674
+ "weight_map": state_dict_split.tensor_to_filename,
675
+ }
676
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
677
+ save_index_file = os.path.join(output_dir, save_index_file)
678
+ with open(save_index_file, "w", encoding="utf-8") as f:
679
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
680
+ f.write(content)
681
+
682
+
683
+ def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
684
+ """
685
+ 1. Put the provided model to cpu
686
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
687
+ 3. Load it into the provided model
688
+
689
+ Args:
690
+ - ``model``: the model object to update
691
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
692
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
693
+
694
+ Returns:
695
+ - ``model`: modified model
696
+
697
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
698
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
699
+ conveniently placed for you in the checkpoint folder.
700
+
701
+ A typical usage might be ::
702
+
703
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
704
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
705
+ # submit to model hub or save the model to share with others
706
+
707
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
708
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
709
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
710
+
711
+ """
712
+ logger.info("Extracting fp32 weights")
713
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
714
+
715
+ logger.info("Overwriting model with fp32 weights")
716
+ model = model.cpu()
717
+ model.load_state_dict(state_dict, strict=False)
718
+
719
+ return model
720
+
721
+
722
+ if __name__ == "__main__":
723
+ parser = argparse.ArgumentParser()
724
+ parser.add_argument("checkpoint_dir",
725
+ type=str,
726
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
727
+ parser.add_argument("output_dir",
728
+ type=str,
729
+ help="directory to the pytorch fp32 state_dict output files"
730
+ "(e.g. path/checkpoint-12-output/)")
731
+ parser.add_argument(
732
+ "--max_shard_size",
733
+ type=str,
734
+ default="5GB",
735
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
736
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
737
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
738
+ "without CPU OOM issues.")
739
+ parser.add_argument(
740
+ "--safe_serialization",
741
+ default=False,
742
+ action='store_true',
743
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
744
+ parser.add_argument("-t",
745
+ "--tag",
746
+ type=str,
747
+ default=None,
748
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
749
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
750
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
751
+ args = parser.parse_args()
752
+
753
+ debug = args.debug
754
+
755
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
756
+ args.output_dir,
757
+ max_shard_size=args.max_shard_size,
758
+ safe_serialization=args.safe_serialization,
759
+ tag=args.tag,
760
+ exclude_frozen_parameters=args.exclude_frozen_parameters)
checkpoint-600/README.md ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: microsoft/Phi-4-mini-instruct
3
+ library_name: peft
4
+ pipeline_tag: text-generation
5
+ tags:
6
+ - axolotl
7
+ - base_model:adapter:microsoft/Phi-4-mini-instruct
8
+ - lora
9
+ - transformers
10
+ ---
11
+
12
+ # Model Card for Model ID
13
+
14
+ <!-- Provide a quick summary of what the model is/does. -->
15
+
16
+
17
+
18
+ ## Model Details
19
+
20
+ ### Model Description
21
+
22
+ <!-- Provide a longer summary of what this model is. -->
23
+
24
+
25
+
26
+ - **Developed by:** [More Information Needed]
27
+ - **Funded by [optional]:** [More Information Needed]
28
+ - **Shared by [optional]:** [More Information Needed]
29
+ - **Model type:** [More Information Needed]
30
+ - **Language(s) (NLP):** [More Information Needed]
31
+ - **License:** [More Information Needed]
32
+ - **Finetuned from model [optional]:** [More Information Needed]
33
+
34
+ ### Model Sources [optional]
35
+
36
+ <!-- Provide the basic links for the model. -->
37
+
38
+ - **Repository:** [More Information Needed]
39
+ - **Paper [optional]:** [More Information Needed]
40
+ - **Demo [optional]:** [More Information Needed]
41
+
42
+ ## Uses
43
+
44
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
45
+
46
+ ### Direct Use
47
+
48
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
49
+
50
+ [More Information Needed]
51
+
52
+ ### Downstream Use [optional]
53
+
54
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
55
+
56
+ [More Information Needed]
57
+
58
+ ### Out-of-Scope Use
59
+
60
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
61
+
62
+ [More Information Needed]
63
+
64
+ ## Bias, Risks, and Limitations
65
+
66
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
67
+
68
+ [More Information Needed]
69
+
70
+ ### Recommendations
71
+
72
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
73
+
74
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
75
+
76
+ ## How to Get Started with the Model
77
+
78
+ Use the code below to get started with the model.
79
+
80
+ [More Information Needed]
81
+
82
+ ## Training Details
83
+
84
+ ### Training Data
85
+
86
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
87
+
88
+ [More Information Needed]
89
+
90
+ ### Training Procedure
91
+
92
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
93
+
94
+ #### Preprocessing [optional]
95
+
96
+ [More Information Needed]
97
+
98
+
99
+ #### Training Hyperparameters
100
+
101
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
102
+
103
+ #### Speeds, Sizes, Times [optional]
104
+
105
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
106
+
107
+ [More Information Needed]
108
+
109
+ ## Evaluation
110
+
111
+ <!-- This section describes the evaluation protocols and provides the results. -->
112
+
113
+ ### Testing Data, Factors & Metrics
114
+
115
+ #### Testing Data
116
+
117
+ <!-- This should link to a Dataset Card if possible. -->
118
+
119
+ [More Information Needed]
120
+
121
+ #### Factors
122
+
123
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
124
+
125
+ [More Information Needed]
126
+
127
+ #### Metrics
128
+
129
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
130
+
131
+ [More Information Needed]
132
+
133
+ ### Results
134
+
135
+ [More Information Needed]
136
+
137
+ #### Summary
138
+
139
+
140
+
141
+ ## Model Examination [optional]
142
+
143
+ <!-- Relevant interpretability work for the model goes here -->
144
+
145
+ [More Information Needed]
146
+
147
+ ## Environmental Impact
148
+
149
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
150
+
151
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
152
+
153
+ - **Hardware Type:** [More Information Needed]
154
+ - **Hours used:** [More Information Needed]
155
+ - **Cloud Provider:** [More Information Needed]
156
+ - **Compute Region:** [More Information Needed]
157
+ - **Carbon Emitted:** [More Information Needed]
158
+
159
+ ## Technical Specifications [optional]
160
+
161
+ ### Model Architecture and Objective
162
+
163
+ [More Information Needed]
164
+
165
+ ### Compute Infrastructure
166
+
167
+ [More Information Needed]
168
+
169
+ #### Hardware
170
+
171
+ [More Information Needed]
172
+
173
+ #### Software
174
+
175
+ [More Information Needed]
176
+
177
+ ## Citation [optional]
178
+
179
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
180
+
181
+ **BibTeX:**
182
+
183
+ [More Information Needed]
184
+
185
+ **APA:**
186
+
187
+ [More Information Needed]
188
+
189
+ ## Glossary [optional]
190
+
191
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
192
+
193
+ [More Information Needed]
194
+
195
+ ## More Information [optional]
196
+
197
+ [More Information Needed]
198
+
199
+ ## Model Card Authors [optional]
200
+
201
+ [More Information Needed]
202
+
203
+ ## Model Card Contact
204
+
205
+ [More Information Needed]
206
+ ### Framework versions
207
+
208
+ - PEFT 0.18.1
checkpoint-600/adapter_config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": null,
6
+ "base_model_name_or_path": "microsoft/Phi-4-mini-instruct",
7
+ "bias": "none",
8
+ "corda_config": null,
9
+ "ensure_weight_tying": false,
10
+ "eva_config": null,
11
+ "exclude_modules": null,
12
+ "fan_in_fan_out": null,
13
+ "inference_mode": true,
14
+ "init_lora_weights": true,
15
+ "layer_replication": null,
16
+ "layers_pattern": null,
17
+ "layers_to_transform": null,
18
+ "loftq_config": {},
19
+ "lora_alpha": 16,
20
+ "lora_bias": false,
21
+ "lora_dropout": 0.05,
22
+ "megatron_config": null,
23
+ "megatron_core": "megatron.core",
24
+ "modules_to_save": null,
25
+ "peft_type": "LORA",
26
+ "peft_version": "0.18.1",
27
+ "qalora_group_size": 16,
28
+ "r": 8,
29
+ "rank_pattern": {},
30
+ "revision": null,
31
+ "target_modules": [
32
+ "k_proj",
33
+ "q_proj",
34
+ "v_proj",
35
+ "o_proj"
36
+ ],
37
+ "target_parameters": [],
38
+ "task_type": "CAUSAL_LM",
39
+ "trainable_token_indices": null,
40
+ "use_dora": false,
41
+ "use_qalora": false,
42
+ "use_rslora": false
43
+ }
checkpoint-600/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:320aae774947cf28fda6d90d218709e645cd5f9c86f0efcb31465acc729b957f
3
+ size 3154216
checkpoint-600/added_tokens.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|/tool_call|>": 200026,
3
+ "<|/tool|>": 200024,
4
+ "<|assistant|>": 200019,
5
+ "<|end|>": 200020,
6
+ "<|system|>": 200022,
7
+ "<|tag|>": 200028,
8
+ "<|tool_call|>": 200025,
9
+ "<|tool_response|>": 200027,
10
+ "<|tool|>": 200023,
11
+ "<|user|>": 200021
12
+ }
checkpoint-600/chat_template.jinja ADDED
@@ -0,0 +1 @@
 
 
1
+ {% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}
checkpoint-600/global_step600/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e842313dd7437966c64fd10baa05da318701259a9dfa75195d874caeda11c4b
3
+ size 18884901