Final model for experiment Indonesian

README.md

@@ -1,7 +1,7 @@
 ---
 library_name: transformers
 tags:
-- Serbian
+- Indonesian
 - generated_from_trainer
 model-index:
 - name: childes-segmentation-100k-gpt2_lm-model
@@ -16,53 +16,53 @@ should probably proofread and complete it, then remove this comment. -->
 This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
 It achieves the following results on the evaluation set:
 - epoch: 4000.0
-- eval_absolute_seg_boundary_fscore_Boundary Prediction: 0.5470
-- eval_absolute_seg_boundary_fscore_Entropy: 0.4522
-- eval_absolute_seg_boundary_fscore_Increase in Boundary Prediction: 0.5803
-- eval_absolute_seg_boundary_fscore_Increase in Entropy: 0.4833
-- eval_absolute_seg_boundary_fscore_Increase in Loss: 0.5784
-- eval_absolute_seg_boundary_fscore_Increase in Rank: 0.6139
-- eval_absolute_seg_boundary_fscore_Loss: 0.4960
-- eval_absolute_seg_boundary_fscore_Majority Vote Cutoff: 0.6259
-- eval_absolute_seg_boundary_fscore_Majority Vote Spike: 0.6291
-- eval_absolute_seg_boundary_fscore_Rank: 0.5341
-- eval_absolute_seg_type_fscore_Boundary Prediction: 0.3005
-- eval_absolute_seg_type_fscore_Entropy: 0.2714
-- eval_absolute_seg_type_fscore_Increase in Boundary Prediction: 0.3422
-- eval_absolute_seg_type_fscore_Increase in Entropy: 0.2764
-- eval_absolute_seg_type_fscore_Increase in Loss: 0.3523
-- eval_absolute_seg_type_fscore_Increase in Rank: 0.3931
-- eval_absolute_seg_type_fscore_Loss: 0.2706
-- eval_absolute_seg_type_fscore_Majority Vote Cutoff: 0.4061
-- eval_absolute_seg_type_fscore_Majority Vote Spike: 0.3590
-- eval_absolute_seg_type_fscore_Rank: 0.2985
-- eval_bpc: 4.5128
-- eval_loss: 3.1280
+- eval_absolute_seg_boundary_fscore_Boundary Prediction: 0.5187
+- eval_absolute_seg_boundary_fscore_Entropy: 0.4017
+- eval_absolute_seg_boundary_fscore_Increase in Boundary Prediction: 0.5413
+- eval_absolute_seg_boundary_fscore_Increase in Entropy: 0.4616
+- eval_absolute_seg_boundary_fscore_Increase in Loss: 0.5806
+- eval_absolute_seg_boundary_fscore_Increase in Rank: 0.6071
+- eval_absolute_seg_boundary_fscore_Loss: 0.4839
+- eval_absolute_seg_boundary_fscore_Majority Vote Cutoff: 0.6298
+- eval_absolute_seg_boundary_fscore_Majority Vote Spike: 0.6048
+- eval_absolute_seg_boundary_fscore_Rank: 0.5122
+- eval_absolute_seg_type_fscore_Boundary Prediction: 0.3513
+- eval_absolute_seg_type_fscore_Entropy: 0.2802
+- eval_absolute_seg_type_fscore_Increase in Boundary Prediction: 0.3813
+- eval_absolute_seg_type_fscore_Increase in Entropy: 0.3222
+- eval_absolute_seg_type_fscore_Increase in Loss: 0.3824
+- eval_absolute_seg_type_fscore_Increase in Rank: 0.4208
+- eval_absolute_seg_type_fscore_Loss: 0.3151
+- eval_absolute_seg_type_fscore_Majority Vote Cutoff: 0.4692
+- eval_absolute_seg_type_fscore_Majority Vote Spike: 0.4276
+- eval_absolute_seg_type_fscore_Rank: 0.3621
+- eval_bpc: 4.2803
+- eval_loss: 2.9669
 - eval_model_preparation_time: 0.0008
-- eval_perplexity: 22.8288
-- eval_runtime: 12.2624
-- eval_samples_per_second: 12.151
-- eval_spike_seg_boundary_fscore_Boundary Prediction: 0.5811
-- eval_spike_seg_boundary_fscore_Entropy: 0.4902
-- eval_spike_seg_boundary_fscore_Increase in Boundary Prediction: 0.5768
-- eval_spike_seg_boundary_fscore_Increase in Entropy: 0.4836
-- eval_spike_seg_boundary_fscore_Increase in Loss: 0.5501
-- eval_spike_seg_boundary_fscore_Increase in Rank: 0.5845
-- eval_spike_seg_boundary_fscore_Loss: 0.5259
-- eval_spike_seg_boundary_fscore_Majority Vote Cutoff: 0.6380
-- eval_spike_seg_boundary_fscore_Majority Vote Spike: 0.6029
-- eval_spike_seg_boundary_fscore_Rank: 0.5890
-- eval_spike_seg_type_fscore_Boundary Prediction: 0.2831
-- eval_spike_seg_type_fscore_Entropy: 0.2583
-- eval_spike_seg_type_fscore_Increase in Boundary Prediction: 0.2799
-- eval_spike_seg_type_fscore_Increase in Entropy: 0.2175
-- eval_spike_seg_type_fscore_Increase in Loss: 0.2712
-- eval_spike_seg_type_fscore_Increase in Rank: 0.3042
-- eval_spike_seg_type_fscore_Loss: 0.2594
-- eval_spike_seg_type_fscore_Majority Vote Cutoff: 0.3492
-- eval_spike_seg_type_fscore_Majority Vote Spike: 0.2747
-- eval_spike_seg_type_fscore_Rank: 0.3375
-- eval_steps_per_second: 0.408
+- eval_perplexity: 19.4309
+- eval_runtime: 23.9581
+- eval_samples_per_second: 5.426
+- eval_spike_seg_boundary_fscore_Boundary Prediction: 0.5220
+- eval_spike_seg_boundary_fscore_Entropy: 0.4477
+- eval_spike_seg_boundary_fscore_Increase in Boundary Prediction: 0.5144
+- eval_spike_seg_boundary_fscore_Increase in Entropy: 0.4307
+- eval_spike_seg_boundary_fscore_Increase in Loss: 0.5371
+- eval_spike_seg_boundary_fscore_Increase in Rank: 0.5614
+- eval_spike_seg_boundary_fscore_Loss: 0.5245
+- eval_spike_seg_boundary_fscore_Majority Vote Cutoff: 0.5996
+- eval_spike_seg_boundary_fscore_Majority Vote Spike: 0.5457
+- eval_spike_seg_boundary_fscore_Rank: 0.5726
+- eval_spike_seg_type_fscore_Boundary Prediction: 0.2919
+- eval_spike_seg_type_fscore_Entropy: 0.2635
+- eval_spike_seg_type_fscore_Increase in Boundary Prediction: 0.2716
+- eval_spike_seg_type_fscore_Increase in Entropy: 0.2197
+- eval_spike_seg_type_fscore_Increase in Loss: 0.2948
+- eval_spike_seg_type_fscore_Increase in Rank: 0.3411
+- eval_spike_seg_type_fscore_Loss: 0.2883
+- eval_spike_seg_type_fscore_Majority Vote Cutoff: 0.4045
+- eval_spike_seg_type_fscore_Majority Vote Spike: 0.2782
+- eval_spike_seg_type_fscore_Rank: 0.3612
+- eval_steps_per_second: 0.209
 - step: 100000
 
 ## Model description

config.json

@@ -27,5 +27,5 @@
   "torch_dtype": "float32",
   "transformers_version": "4.44.2",
   "use_cache": true,
-  "vocab_size": 34
+  "vocab_size": 31
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:efe533efbd0b31c095b2145cb7c348667d0ec3699e849cc3981aa0d82694b3c0
-size 2532632
+oid sha256:84fe774c468a033f724e41b8a5406299e48e82cccf7326ca4da90e906360fbcc
+size 2531096

tokenizer.json

@@ -115,36 +115,33 @@
       "PAD": 1,
       "WORD_BOUNDARY": 2,
       "UTT_BOUNDARY": 3,
-      "j": 4,
-      "e̞": 5,
-      "s̪̻": 6,
-      "t̪̻": 7,
-      "u": 8,
-      "l": 9,
-      "o̞": 10,
-      "ʒ̺": 11,
-      "i": 12,
-      "ʋ": 13,
-      "d̪̻": 14,
-      "ä": 15,
-      "m": 16,
-      "n": 17,
-      "r": 18,
-      "k": 19,
-      "t̪̻s̪̻": 20,
+      "s": 4,
+      "i": 5,
+      "n": 6,
+      "m": 7,
+      "a": 8,
+      "j": 9,
+      "u": 10,
+      "k": 11,
+      "o": 12,
+      "h": 13,
+      "l": 14,
+      "t": 15,
+      "w": 16,
+      "d̠ʒ": 17,
+      "ŋ": 18,
+      "ə": 19,
+      "d": 20,
       "p": 21,
-      "ʃ̺": 22,
-      "x": 23,
-      "b": 24,
-      "ɡ": 25,
-      "t̻ʃ̻": 26,
+      "ɡ": 22,
+      "b": 23,
+      "r": 24,
+      "ɲ": 25,
+      "t̠ʃ": 26,
       "f": 27,
-      "z̪̻": 28,
-      "ɲ": 29,
-      "ʎ": 30,
-      "d̻ʒ̻": 31,
-      "y": 32,
-      "w": 33
+      "z": 28,
+      "ʃ": 29,
+      "x": 30
     },
     "unk_token": "UNK"
   }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52482822ae0aead84723e50320eafa35fe04b4377ff02a99470964996e89b1f0
+oid sha256:0948cb11c8ef3cbbac150d6b065e41c48eafcb71430ca203d424013a0dc9d431
 size 5368

vocab.json

@@ -1 +1 @@
-{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"j":4,"e̞":5,"s̪̻":6,"t̪̻":7,"u":8,"l":9,"o̞":10,"ʒ̺":11,"i":12,"ʋ":13,"d̪̻":14,"ä":15,"m":16,"n":17,"r":18,"k":19,"t̪̻s̪̻":20,"p":21,"ʃ̺":22,"x":23,"b":24,"ɡ":25,"t̻ʃ̻":26,"f":27,"z̪̻":28,"ɲ":29,"ʎ":30,"d̻ʒ̻":31,"y":32,"w":33}
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"s":4,"i":5,"n":6,"m":7,"a":8,"j":9,"u":10,"k":11,"o":12,"h":13,"l":14,"t":15,"w":16,"d̠ʒ":17,"ŋ":18,"ə":19,"d":20,"p":21,"ɡ":22,"b":23,"r":24,"ɲ":25,"t̠ʃ":26,"f":27,"z":28,"ʃ":29,"x":30}