Final model for experiment Mandarin

README.md

@@ -1,7 +1,7 @@
 ---
 library_name: transformers
 tags:
-- Estonian
+- Mandarin
 - generated_from_trainer
 model-index:
 - name: childes-segmentation-100k-gpt2_lm-model
@@ -16,53 +16,53 @@ should probably proofread and complete it, then remove this comment. -->
 This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
 It achieves the following results on the evaluation set:
 - epoch: 4000.0
-- eval_absolute_seg_boundary_fscore_Boundary Prediction: 0.5173
-- eval_absolute_seg_boundary_fscore_Entropy: 0.4122
-- eval_absolute_seg_boundary_fscore_Increase in Boundary Prediction: 0.5774
-- eval_absolute_seg_boundary_fscore_Increase in Entropy: 0.4825
-- eval_absolute_seg_boundary_fscore_Increase in Loss: 0.5014
-- eval_absolute_seg_boundary_fscore_Increase in Rank: 0.5198
-- eval_absolute_seg_boundary_fscore_Loss: 0.4373
-- eval_absolute_seg_boundary_fscore_Majority Vote Cutoff: 0.5816
-- eval_absolute_seg_boundary_fscore_Majority Vote Spike: 0.5640
-- eval_absolute_seg_boundary_fscore_Rank: 0.4601
-- eval_absolute_seg_type_fscore_Boundary Prediction: 0.3117
-- eval_absolute_seg_type_fscore_Entropy: 0.2019
-- eval_absolute_seg_type_fscore_Increase in Boundary Prediction: 0.3048
-- eval_absolute_seg_type_fscore_Increase in Entropy: 0.2221
-- eval_absolute_seg_type_fscore_Increase in Loss: 0.2382
-- eval_absolute_seg_type_fscore_Increase in Rank: 0.2725
-- eval_absolute_seg_type_fscore_Loss: 0.1844
-- eval_absolute_seg_type_fscore_Majority Vote Cutoff: 0.3083
-- eval_absolute_seg_type_fscore_Majority Vote Spike: 0.2870
-- eval_absolute_seg_type_fscore_Rank: 0.2212
-- eval_bpc: 4.5092
-- eval_loss: 3.1255
+- eval_absolute_seg_boundary_fscore_Boundary Prediction: 0.4895
+- eval_absolute_seg_boundary_fscore_Entropy: 0.5708
+- eval_absolute_seg_boundary_fscore_Increase in Boundary Prediction: 0.4915
+- eval_absolute_seg_boundary_fscore_Increase in Entropy: 0.6177
+- eval_absolute_seg_boundary_fscore_Increase in Loss: 0.6662
+- eval_absolute_seg_boundary_fscore_Increase in Rank: 0.7024
+- eval_absolute_seg_boundary_fscore_Loss: 0.5977
+- eval_absolute_seg_boundary_fscore_Majority Vote Cutoff: 0.7211
+- eval_absolute_seg_boundary_fscore_Majority Vote Spike: 0.7641
+- eval_absolute_seg_boundary_fscore_Rank: 0.6237
+- eval_absolute_seg_type_fscore_Boundary Prediction: 0.4682
+- eval_absolute_seg_type_fscore_Entropy: 0.3143
+- eval_absolute_seg_type_fscore_Increase in Boundary Prediction: 0.5365
+- eval_absolute_seg_type_fscore_Increase in Entropy: 0.3262
+- eval_absolute_seg_type_fscore_Increase in Loss: 0.3540
+- eval_absolute_seg_type_fscore_Increase in Rank: 0.4184
+- eval_absolute_seg_type_fscore_Loss: 0.2843
+- eval_absolute_seg_type_fscore_Majority Vote Cutoff: 0.4790
+- eval_absolute_seg_type_fscore_Majority Vote Spike: 0.4732
+- eval_absolute_seg_type_fscore_Rank: 0.3459
+- eval_bpc: 4.5280
+- eval_loss: 3.1386
 - eval_model_preparation_time: 0.0008
-- eval_perplexity: 22.7722
-- eval_runtime: 11.7267
-- eval_samples_per_second: 16.884
-- eval_spike_seg_boundary_fscore_Boundary Prediction: 0.6098
-- eval_spike_seg_boundary_fscore_Entropy: 0.4166
-- eval_spike_seg_boundary_fscore_Increase in Boundary Prediction: 0.6078
-- eval_spike_seg_boundary_fscore_Increase in Entropy: 0.4578
-- eval_spike_seg_boundary_fscore_Increase in Loss: 0.4777
-- eval_spike_seg_boundary_fscore_Increase in Rank: 0.5160
-- eval_spike_seg_boundary_fscore_Loss: 0.4197
-- eval_spike_seg_boundary_fscore_Majority Vote Cutoff: 0.5653
-- eval_spike_seg_boundary_fscore_Majority Vote Spike: 0.5421
-- eval_spike_seg_boundary_fscore_Rank: 0.4477
-- eval_spike_seg_type_fscore_Boundary Prediction: 0.3007
-- eval_spike_seg_type_fscore_Entropy: 0.1823
-- eval_spike_seg_type_fscore_Increase in Boundary Prediction: 0.2908
-- eval_spike_seg_type_fscore_Increase in Entropy: 0.1909
-- eval_spike_seg_type_fscore_Increase in Loss: 0.1929
-- eval_spike_seg_type_fscore_Increase in Rank: 0.2341
-- eval_spike_seg_type_fscore_Loss: 0.1525
-- eval_spike_seg_type_fscore_Majority Vote Cutoff: 0.2721
-- eval_spike_seg_type_fscore_Majority Vote Spike: 0.2019
-- eval_spike_seg_type_fscore_Rank: 0.1958
-- eval_steps_per_second: 0.597
+- eval_perplexity: 23.0705
+- eval_runtime: 45.72
+- eval_samples_per_second: 3.084
+- eval_spike_seg_boundary_fscore_Boundary Prediction: 0.8113
+- eval_spike_seg_boundary_fscore_Entropy: 0.5823
+- eval_spike_seg_boundary_fscore_Increase in Boundary Prediction: 0.8144
+- eval_spike_seg_boundary_fscore_Increase in Entropy: 0.6192
+- eval_spike_seg_boundary_fscore_Increase in Loss: 0.6641
+- eval_spike_seg_boundary_fscore_Increase in Rank: 0.7155
+- eval_spike_seg_boundary_fscore_Loss: 0.6074
+- eval_spike_seg_boundary_fscore_Majority Vote Cutoff: 0.7518
+- eval_spike_seg_boundary_fscore_Majority Vote Spike: 0.7574
+- eval_spike_seg_boundary_fscore_Rank: 0.6497
+- eval_spike_seg_type_fscore_Boundary Prediction: 0.5335
+- eval_spike_seg_type_fscore_Entropy: 0.3071
+- eval_spike_seg_type_fscore_Increase in Boundary Prediction: 0.5314
+- eval_spike_seg_type_fscore_Increase in Entropy: 0.3199
+- eval_spike_seg_type_fscore_Increase in Loss: 0.3475
+- eval_spike_seg_type_fscore_Increase in Rank: 0.4171
+- eval_spike_seg_type_fscore_Loss: 0.2965
+- eval_spike_seg_type_fscore_Majority Vote Cutoff: 0.4907
+- eval_spike_seg_type_fscore_Majority Vote Spike: 0.4385
+- eval_spike_seg_type_fscore_Rank: 0.3780
+- eval_steps_per_second: 0.109
 - step: 100000
 
 ## Model description

config.json

@@ -27,5 +27,5 @@
   "torch_dtype": "float32",
   "transformers_version": "4.44.2",
   "use_cache": true,
-  "vocab_size": 68
+  "vocab_size": 115
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d028c1e5ff80b0b3b65eb47e07892265643b1df9517383cdcc42fcbf07c0bcb
-size 2550040
+oid sha256:1f7efa21a386251f84b729d3205aa549d7c8926a03a23ff83f171997f706fda2
+size 2574104

tokenizer.json

@@ -51,7 +51,7 @@
     ]
   },
   "pre_tokenizer": {
-    "type": "Whitespace"
+    "type": "WhitespaceSplit"
   },
   "post_processor": {
     "type": "TemplateProcessing",
@@ -115,70 +115,117 @@
       "PAD": 1,
       "WORD_BOUNDARY": 2,
       "UTT_BOUNDARY": 3,
-      "n": 4,
-      "o": 5,
-      "t": 6,
-      "ʃ": 7,
-      "a": 8,
-      "uː": 9,
-      "m": 10,
-      "u": 11,
-      "tʲ": 12,
-      "i": 13,
-      "s": 14,
-      "eː": 15,
-      "d": 16,
-      "iː": 17,
-      "k": 18,
-      "ɡ": 19,
-      "ɑ": 20,
-      "ɤ": 21,
-      "ʊ": 22,
-      "sʲ": 23,
-      "j": 24,
-      "aː": 25,
-      "h": 26,
-      "v": 27,
-      "æi": 28,
-      "kː": 29,
-      "e": 30,
-      "ɪ": 31,
-      "tː": 32,
-      "r": 33,
-      "ɛ": 34,
-      "mː": 35,
-      "p": 36,
-      "sː": 37,
-      "æ": 38,
-      "l": 39,
-      "pː": 40,
-      "yː": 41,
-      "æː": 42,
-      "b": 43,
-      "ɔ": 44,
-      "ɤː": 45,
-      "lː": 46,
-      "ø": 47,
-      "øː": 48,
-      "ŋ": 49,
-      "y": 50,
-      "oː": 51,
-      "rː": 52,
-      "ɲ": 53,
-      "nː": 54,
-      "w": 55,
-      "tʲː": 56,
-      "øɪ̯": 57,
-      "f": 58,
-      "dʲ": 59,
-      "sʲː": 60,
-      "t̠ʃ": 61,
-      "ʃː": 62,
-      "ʒ": 63,
-      "z": 64,
-      "fː": 65,
-      "dː": 66,
-      "yi": 67
+      "a˧˥": 4,
+      "u˧˥": 5,
+      "a˥": 6,
+      "au": 7,
+      "n": 8,
+      "a˥˩": 9,
+      "ʃ̺": 10,
+      "ɻ̩˥˩": 11,
+      "ə˧˥": 12,
+      "m": 13,
+      "ɤ": 14,
+      "p": 15,
+      "j": 16,
+      "e˧˥": 17,
+      "kʰ": 18,
+      "k": 19,
+      "ɤ˥˩": 20,
+      "w": 21,
+      "o˥": 22,
+      "t̠ʃ̺ʰ": 23,
+      "ə˥": 24,
+      "ŋ": 25,
+      "t": 26,
+      "ʊ˥": 27,
+      "ɕ": 28,
+      "i": 29,
+      "a": 30,
+      "l": 31,
+      "au˧˩˧": 32,
+      "x": 33,
+      "u˧˩˧": 34,
+      "i˥": 35,
+      "ei˧˩˧": 36,
+      "pʰ": 37,
+      "i˧˥": 38,
+      "ai˧˥": 39,
+      "ou˧˩˧": 40,
+      "ɤ˧˥": 41,
+      "o˧˩˧": 42,
+      "tɕ": 43,
+      "au˥˩": 44,
+      "ts": 45,
+      "ə˧˩˧": 46,
+      "ɤ˥": 47,
+      "ei˧˥": 48,
+      "ʊ˧˥": 49,
+      "i˧˩˧": 50,
+      "t̠ʃ̺": 51,
+      "ɻ̩˧˩˧": 52,
+      "ei˥˩": 53,
+      "s": 54,
+      "u˥˩": 55,
+      "ɹ̪̩": 56,
+      "ai˥": 57,
+      "u˥": 58,
+      "tɕʰ": 59,
+      "a˧˩˧": 60,
+      "ai˥˩": 61,
+      "ɛ˥˩": 62,
+      "f": 63,
+      "i˥˩": 64,
+      "y˥˩": 65,
+      "au˧˥": 66,
+      "ɻ": 67,
+      "ou˥˩": 68,
+      "e˥": 69,
+      "tʰ": 70,
+      "ɹ̪̩˥˩": 71,
+      "ɛ˧˥": 72,
+      "au˥": 73,
+      "ou˧˥": 74,
+      "e˧˩˧": 75,
+      "ɛ˥": 76,
+      "ɻ̩˥": 77,
+      "ɥ": 78,
+      "ɹ̪̩˧˩˧": 79,
+      "ai˧˩˧": 80,
+      "ou˥": 81,
+      "o˥˩": 82,
+      "ɛ˧˩˧": 83,
+      "ʊ˧˩˧": 84,
+      "ɔ˥": 85,
+      "tsʰ": 86,
+      "ei": 87,
+      "ə˥˩": 88,
+      "o": 89,
+      "ʊ˥˩": 90,
+      "ou": 91,
+      "ɤ˧˩˧": 92,
+      "o˧˥": 93,
+      "ei˥": 94,
+      "e˥˩": 95,
+      "ɚ˧˩˧": 96,
+      "y˥": 97,
+      "ɚ˥˩": 98,
+      "y˧˥": 99,
+      "ɻ̩": 100,
+      "y˧˩˧": 101,
+      "ɹ̪̩˥": 102,
+      "ɻ̩˧˥": 103,
+      "u": 104,
+      "ə": 105,
+      "ai": 106,
+      "ʊ": 107,
+      "e": 108,
+      "ɚ˧˥": 109,
+      "ɔ˥˩": 110,
+      "ɹ̪̩˧˥": 111,
+      "ɛ": 112,
+      "y": 113,
+      "m˧˥": 114
     },
     "unk_token": "UNK"
   }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de9f2ff350f6b9b053d00cc795964f932021ce06c9925b1fc687b3465fc34f31
+oid sha256:ab5aeece3087b363ef297505f6884ea73978b1fc89a8380b1aea665daf958b97
 size 5368

vocab.json

@@ -1 +1 @@
-{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"n":4,"o":5,"t":6,"ʃ":7,"a":8,"uː":9,"m":10,"u":11,"tʲ":12,"i":13,"s":14,"eː":15,"d":16,"iː":17,"k":18,"ɡ":19,"ɑ":20,"ɤ":21,"ʊ":22,"sʲ":23,"j":24,"aː":25,"h":26,"v":27,"æi":28,"kː":29,"e":30,"ɪ":31,"tː":32,"r":33,"ɛ":34,"mː":35,"p":36,"sː":37,"æ":38,"l":39,"pː":40,"yː":41,"æː":42,"b":43,"ɔ":44,"ɤː":45,"lː":46,"ø":47,"øː":48,"ŋ":49,"y":50,"oː":51,"rː":52,"ɲ":53,"nː":54,"w":55,"tʲː":56,"øɪ̯":57,"f":58,"dʲ":59,"sʲː":60,"t̠ʃ":61,"ʃː":62,"ʒ":63,"z":64,"fː":65,"dː":66,"yi":67}
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"a˧˥":4,"u˧˥":5,"a˥":6,"au":7,"n":8,"a˥˩":9,"ʃ̺":10,"ɻ̩˥˩":11,"ə˧˥":12,"m":13,"ɤ":14,"p":15,"j":16,"e˧˥":17,"kʰ":18,"k":19,"ɤ˥˩":20,"w":21,"o˥":22,"t̠ʃ̺ʰ":23,"ə˥":24,"ŋ":25,"t":26,"ʊ˥":27,"ɕ":28,"i":29,"a":30,"l":31,"au˧˩˧":32,"x":33,"u˧˩˧":34,"i˥":35,"ei˧˩˧":36,"pʰ":37,"i˧˥":38,"ai˧˥":39,"ou˧˩˧":40,"ɤ˧˥":41,"o˧˩˧":42,"tɕ":43,"au˥˩":44,"ts":45,"ə˧˩˧":46,"ɤ˥":47,"ei˧˥":48,"ʊ˧˥":49,"i˧˩˧":50,"t̠ʃ̺":51,"ɻ̩˧˩˧":52,"ei˥˩":53,"s":54,"u˥˩":55,"ɹ̪̩":56,"ai˥":57,"u˥":58,"tɕʰ":59,"a˧˩˧":60,"ai˥˩":61,"ɛ˥˩":62,"f":63,"i˥˩":64,"y˥˩":65,"au˧˥":66,"ɻ":67,"ou˥˩":68,"e˥":69,"tʰ":70,"ɹ̪̩˥˩":71,"ɛ˧˥":72,"au˥":73,"ou˧˥":74,"e˧˩˧":75,"ɛ˥":76,"ɻ̩˥":77,"ɥ":78,"ɹ̪̩˧˩˧":79,"ai˧˩˧":80,"ou˥":81,"o˥˩":82,"ɛ˧˩˧":83,"ʊ˧˩˧":84,"ɔ˥":85,"tsʰ":86,"ei":87,"ə˥˩":88,"o":89,"ʊ˥˩":90,"ou":91,"ɤ˧˩˧":92,"o˧˥":93,"ei˥":94,"e˥˩":95,"ɚ˧˩˧":96,"y˥":97,"ɚ˥˩":98,"y˧˥":99,"ɻ̩":100,"y˧˩˧":101,"ɹ̪̩˥":102,"ɻ̩˧˥":103,"u":104,"ə":105,"ai":106,"ʊ":107,"e":108,"ɚ˧˥":109,"ɔ˥˩":110,"ɹ̪̩˧˥":111,"ɛ":112,"y":113,"m˧˥":114}