HF中国镜像站's logo

Grogros
/
dmWM-LLama-3-1B-Harm-ft-HarmData-AlpacaGPT4-OpenWebText-d4-a0.25

Text Generation
Transformers
TensorBoard
Safetensors
llama
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Metrics Community
Grogros commited on
Commit
3343fa9
·
verified ·
1 Parent(s): da058bc

Training in progress, step 500, checkpoint

Browse files
Files changed (8) hide show
  1. checkpoint-500/config.json +40 -0
  2. checkpoint-500/generation_config.json +12 -0
  3. checkpoint-500/model.safetensors +3 -0
  4. checkpoint-500/optimizer.pt +3 -0
  5. checkpoint-500/rng_state.pth +3 -0
  6. checkpoint-500/scheduler.pt +3 -0
  7. checkpoint-500/trainer_state.json +383 -0
  8. checkpoint-500/training_args.bin +3 -0
checkpoint-500/config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "huihui-ai/Llama-3.2-1B-Instruct-abliterated",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "eos_token_id": [
10
+ 128001,
11
+ 128008,
12
+ 128009
13
+ ],
14
+ "head_dim": 64,
15
+ "hidden_act": "silu",
16
+ "hidden_size": 2048,
17
+ "initializer_range": 0.02,
18
+ "intermediate_size": 8192,
19
+ "max_position_embeddings": 131072,
20
+ "mlp_bias": false,
21
+ "model_type": "llama",
22
+ "num_attention_heads": 32,
23
+ "num_hidden_layers": 16,
24
+ "num_key_value_heads": 8,
25
+ "pretraining_tp": 1,
26
+ "rms_norm_eps": 1e-05,
27
+ "rope_scaling": {
28
+ "factor": 32.0,
29
+ "high_freq_factor": 4.0,
30
+ "low_freq_factor": 1.0,
31
+ "original_max_position_embeddings": 8192,
32
+ "rope_type": "llama3"
33
+ },
34
+ "rope_theta": 500000.0,
35
+ "tie_word_embeddings": true,
36
+ "torch_dtype": "bfloat16",
37
+ "transformers_version": "4.46.3",
38
+ "use_cache": true,
39
+ "vocab_size": 128256
40
+ }
checkpoint-500/generation_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 128000,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 128001,
6
+ 128008,
7
+ 128009
8
+ ],
9
+ "temperature": 0.6,
10
+ "top_p": 0.9,
11
+ "transformers_version": "4.46.3"
12
+ }
checkpoint-500/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:11a44d46c410d46c1d3a2af1202cf7c2113fa3cbd0ab742e5409e8f93955a23e
3
+ size 2471645608
checkpoint-500/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68ee55aad6cb7febbd13e867ed69ae22c5f1e1eac950d42c097c2789fc0f0d46
3
+ size 3724602
checkpoint-500/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a56f2618a2cc50a494d72bb90d568c6c748a64872c9527423c54ff20d17de02d
3
+ size 14244
checkpoint-500/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:31f20d3fd8f2ca072c88294e1b245d2fc0feec4c84032b4ae4d3711181320962
3
+ size 1064
checkpoint-500/trainer_state.json ADDED
@@ -0,0 +1,383 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.616088947024199,
5
+ "eval_steps": 500,
6
+ "global_step": 500,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.052321778940483975,
13
+ "grad_norm": 105.5,
14
+ "learning_rate": 8.000000000000001e-07,
15
+ "loss": 0.5046,
16
+ "step": 10
17
+ },
18
+ {
19
+ "epoch": 0.10464355788096795,
20
+ "grad_norm": 332.0,
21
+ "learning_rate": 1.6000000000000001e-06,
22
+ "loss": 0.5118,
23
+ "step": 20
24
+ },
25
+ {
26
+ "epoch": 0.15696533682145192,
27
+ "grad_norm": 776.0,
28
+ "learning_rate": 2.4000000000000003e-06,
29
+ "loss": 0.5878,
30
+ "step": 30
31
+ },
32
+ {
33
+ "epoch": 0.2092871157619359,
34
+ "grad_norm": 712.0,
35
+ "learning_rate": 3.2000000000000003e-06,
36
+ "loss": 0.7578,
37
+ "step": 40
38
+ },
39
+ {
40
+ "epoch": 0.2616088947024199,
41
+ "grad_norm": 480.0,
42
+ "learning_rate": 4.000000000000001e-06,
43
+ "loss": 0.9917,
44
+ "step": 50
45
+ },
46
+ {
47
+ "epoch": 0.31393067364290383,
48
+ "grad_norm": 2672.0,
49
+ "learning_rate": 4.800000000000001e-06,
50
+ "loss": 1.4457,
51
+ "step": 60
52
+ },
53
+ {
54
+ "epoch": 0.36625245258338784,
55
+ "grad_norm": 400.0,
56
+ "learning_rate": 5.600000000000001e-06,
57
+ "loss": 1.1678,
58
+ "step": 70
59
+ },
60
+ {
61
+ "epoch": 0.4185742315238718,
62
+ "grad_norm": 426.0,
63
+ "learning_rate": 6.4000000000000006e-06,
64
+ "loss": 1.7799,
65
+ "step": 80
66
+ },
67
+ {
68
+ "epoch": 0.4708960104643558,
69
+ "grad_norm": 470.0,
70
+ "learning_rate": 7.2000000000000005e-06,
71
+ "loss": 1.7215,
72
+ "step": 90
73
+ },
74
+ {
75
+ "epoch": 0.5232177894048398,
76
+ "grad_norm": 644.0,
77
+ "learning_rate": 8.000000000000001e-06,
78
+ "loss": 1.5971,
79
+ "step": 100
80
+ },
81
+ {
82
+ "epoch": 0.5755395683453237,
83
+ "grad_norm": 752.0,
84
+ "learning_rate": 8.8e-06,
85
+ "loss": 1.5786,
86
+ "step": 110
87
+ },
88
+ {
89
+ "epoch": 0.6278613472858077,
90
+ "grad_norm": 540.0,
91
+ "learning_rate": 9.600000000000001e-06,
92
+ "loss": 1.3361,
93
+ "step": 120
94
+ },
95
+ {
96
+ "epoch": 0.6801831262262917,
97
+ "grad_norm": 92.0,
98
+ "learning_rate": 1.04e-05,
99
+ "loss": 1.2161,
100
+ "step": 130
101
+ },
102
+ {
103
+ "epoch": 0.7325049051667757,
104
+ "grad_norm": 130.0,
105
+ "learning_rate": 1.1200000000000001e-05,
106
+ "loss": 1.1037,
107
+ "step": 140
108
+ },
109
+ {
110
+ "epoch": 0.7848266841072596,
111
+ "grad_norm": 111.0,
112
+ "learning_rate": 1.2e-05,
113
+ "loss": 0.9343,
114
+ "step": 150
115
+ },
116
+ {
117
+ "epoch": 0.8371484630477436,
118
+ "grad_norm": 280.0,
119
+ "learning_rate": 1.2800000000000001e-05,
120
+ "loss": 1.3136,
121
+ "step": 160
122
+ },
123
+ {
124
+ "epoch": 0.8894702419882276,
125
+ "grad_norm": 45.5,
126
+ "learning_rate": 1.3600000000000002e-05,
127
+ "loss": 1.1424,
128
+ "step": 170
129
+ },
130
+ {
131
+ "epoch": 0.9417920209287116,
132
+ "grad_norm": 45.5,
133
+ "learning_rate": 1.4400000000000001e-05,
134
+ "loss": 1.0245,
135
+ "step": 180
136
+ },
137
+ {
138
+ "epoch": 0.9941137998691956,
139
+ "grad_norm": 37.25,
140
+ "learning_rate": 1.5200000000000002e-05,
141
+ "loss": 0.8959,
142
+ "step": 190
143
+ },
144
+ {
145
+ "epoch": 1.0464355788096795,
146
+ "grad_norm": 54.25,
147
+ "learning_rate": 1.6000000000000003e-05,
148
+ "loss": 1.1183,
149
+ "step": 200
150
+ },
151
+ {
152
+ "epoch": 1.0987573577501635,
153
+ "grad_norm": 181.0,
154
+ "learning_rate": 1.6800000000000002e-05,
155
+ "loss": 1.0717,
156
+ "step": 210
157
+ },
158
+ {
159
+ "epoch": 1.1510791366906474,
160
+ "grad_norm": 111.0,
161
+ "learning_rate": 1.76e-05,
162
+ "loss": 1.5705,
163
+ "step": 220
164
+ },
165
+ {
166
+ "epoch": 1.2034009156311314,
167
+ "grad_norm": 41.75,
168
+ "learning_rate": 1.8400000000000003e-05,
169
+ "loss": 0.8357,
170
+ "step": 230
171
+ },
172
+ {
173
+ "epoch": 1.2557226945716153,
174
+ "grad_norm": 43.5,
175
+ "learning_rate": 1.9200000000000003e-05,
176
+ "loss": 0.9431,
177
+ "step": 240
178
+ },
179
+ {
180
+ "epoch": 1.3080444735120995,
181
+ "grad_norm": 47.0,
182
+ "learning_rate": 2e-05,
183
+ "loss": 0.7984,
184
+ "step": 250
185
+ },
186
+ {
187
+ "epoch": 1.3603662524525835,
188
+ "grad_norm": 35.0,
189
+ "learning_rate": 1.9999025240093045e-05,
190
+ "loss": 0.7773,
191
+ "step": 260
192
+ },
193
+ {
194
+ "epoch": 1.4126880313930674,
195
+ "grad_norm": 32.0,
196
+ "learning_rate": 1.9996101150403543e-05,
197
+ "loss": 0.8533,
198
+ "step": 270
199
+ },
200
+ {
201
+ "epoch": 1.4650098103335514,
202
+ "grad_norm": 19.25,
203
+ "learning_rate": 1.9991228300988586e-05,
204
+ "loss": 0.7371,
205
+ "step": 280
206
+ },
207
+ {
208
+ "epoch": 1.5173315892740353,
209
+ "grad_norm": 33.25,
210
+ "learning_rate": 1.9984407641819812e-05,
211
+ "loss": 0.6932,
212
+ "step": 290
213
+ },
214
+ {
215
+ "epoch": 1.5696533682145193,
216
+ "grad_norm": 22.375,
217
+ "learning_rate": 1.9975640502598243e-05,
218
+ "loss": 0.7702,
219
+ "step": 300
220
+ },
221
+ {
222
+ "epoch": 1.6219751471550032,
223
+ "grad_norm": 14.0625,
224
+ "learning_rate": 1.9964928592495046e-05,
225
+ "loss": 0.6627,
226
+ "step": 310
227
+ },
228
+ {
229
+ "epoch": 1.6742969260954872,
230
+ "grad_norm": 28.75,
231
+ "learning_rate": 1.9952273999818312e-05,
232
+ "loss": 0.694,
233
+ "step": 320
234
+ },
235
+ {
236
+ "epoch": 1.7266187050359711,
237
+ "grad_norm": 23.0,
238
+ "learning_rate": 1.9937679191605964e-05,
239
+ "loss": 0.5974,
240
+ "step": 330
241
+ },
242
+ {
243
+ "epoch": 1.7789404839764553,
244
+ "grad_norm": 33.5,
245
+ "learning_rate": 1.9921147013144782e-05,
246
+ "loss": 0.6176,
247
+ "step": 340
248
+ },
249
+ {
250
+ "epoch": 1.831262262916939,
251
+ "grad_norm": 34.75,
252
+ "learning_rate": 1.9902680687415704e-05,
253
+ "loss": 0.5954,
254
+ "step": 350
255
+ },
256
+ {
257
+ "epoch": 1.8835840418574232,
258
+ "grad_norm": 25.125,
259
+ "learning_rate": 1.988228381446553e-05,
260
+ "loss": 0.5873,
261
+ "step": 360
262
+ },
263
+ {
264
+ "epoch": 1.9359058207979072,
265
+ "grad_norm": 18.25,
266
+ "learning_rate": 1.985996037070505e-05,
267
+ "loss": 0.603,
268
+ "step": 370
269
+ },
270
+ {
271
+ "epoch": 1.9882275997383911,
272
+ "grad_norm": 16.375,
273
+ "learning_rate": 1.983571470813386e-05,
274
+ "loss": 0.57,
275
+ "step": 380
276
+ },
277
+ {
278
+ "epoch": 2.040549378678875,
279
+ "grad_norm": 21.875,
280
+ "learning_rate": 1.9809551553491918e-05,
281
+ "loss": 0.5213,
282
+ "step": 390
283
+ },
284
+ {
285
+ "epoch": 2.092871157619359,
286
+ "grad_norm": 30.5,
287
+ "learning_rate": 1.9781476007338058e-05,
288
+ "loss": 0.5187,
289
+ "step": 400
290
+ },
291
+ {
292
+ "epoch": 2.145192936559843,
293
+ "grad_norm": 32.0,
294
+ "learning_rate": 1.9751493543055634e-05,
295
+ "loss": 0.549,
296
+ "step": 410
297
+ },
298
+ {
299
+ "epoch": 2.197514715500327,
300
+ "grad_norm": 26.875,
301
+ "learning_rate": 1.9719610005785466e-05,
302
+ "loss": 0.5099,
303
+ "step": 420
304
+ },
305
+ {
306
+ "epoch": 2.249836494440811,
307
+ "grad_norm": 25.625,
308
+ "learning_rate": 1.9685831611286312e-05,
309
+ "loss": 0.5062,
310
+ "step": 430
311
+ },
312
+ {
313
+ "epoch": 2.302158273381295,
314
+ "grad_norm": 22.125,
315
+ "learning_rate": 1.9650164944723116e-05,
316
+ "loss": 0.5366,
317
+ "step": 440
318
+ },
319
+ {
320
+ "epoch": 2.354480052321779,
321
+ "grad_norm": 16.625,
322
+ "learning_rate": 1.961261695938319e-05,
323
+ "loss": 0.5087,
324
+ "step": 450
325
+ },
326
+ {
327
+ "epoch": 2.4068018312622628,
328
+ "grad_norm": 13.0625,
329
+ "learning_rate": 1.9573194975320672e-05,
330
+ "loss": 0.5254,
331
+ "step": 460
332
+ },
333
+ {
334
+ "epoch": 2.459123610202747,
335
+ "grad_norm": 21.25,
336
+ "learning_rate": 1.9531906677929472e-05,
337
+ "loss": 0.5826,
338
+ "step": 470
339
+ },
340
+ {
341
+ "epoch": 2.5114453891432307,
342
+ "grad_norm": 17.0,
343
+ "learning_rate": 1.9488760116444966e-05,
344
+ "loss": 0.4715,
345
+ "step": 480
346
+ },
347
+ {
348
+ "epoch": 2.563767168083715,
349
+ "grad_norm": 43.5,
350
+ "learning_rate": 1.944376370237481e-05,
351
+ "loss": 0.5049,
352
+ "step": 490
353
+ },
354
+ {
355
+ "epoch": 2.616088947024199,
356
+ "grad_norm": 16.75,
357
+ "learning_rate": 1.9396926207859085e-05,
358
+ "loss": 0.4362,
359
+ "step": 500
360
+ }
361
+ ],
362
+ "logging_steps": 10,
363
+ "max_steps": 2500,
364
+ "num_input_tokens_seen": 0,
365
+ "num_train_epochs": 14,
366
+ "save_steps": 500,
367
+ "stateful_callbacks": {
368
+ "TrainerControl": {
369
+ "args": {
370
+ "should_epoch_stop": false,
371
+ "should_evaluate": false,
372
+ "should_log": false,
373
+ "should_save": true,
374
+ "should_training_stop": false
375
+ },
376
+ "attributes": {}
377
+ }
378
+ },
379
+ "total_flos": 1.9128047871039898e+17,
380
+ "train_batch_size": 4,
381
+ "trial_name": null,
382
+ "trial_params": null
383
+ }
checkpoint-500/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3d21c6f2fb439d94b5c36d9d8ef382d1f1d2972cfe7969294a441f3bebef36c
3
+ size 5368