{ | |
"version": "1.0", | |
"truncation": null, | |
"padding": null, | |
"added_tokens": [ | |
{ | |
"id": 0, | |
"content": "UNK", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 1, | |
"content": "PAD", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 2, | |
"content": "WORD_BOUNDARY", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 3, | |
"content": "UTT_BOUNDARY", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
} | |
], | |
"normalizer": { | |
"type": "Sequence", | |
"normalizers": [ | |
{ | |
"type": "Strip", | |
"strip_left": true, | |
"strip_right": true | |
} | |
] | |
}, | |
"pre_tokenizer": { | |
"type": "Whitespace" | |
}, | |
"post_processor": { | |
"type": "TemplateProcessing", | |
"single": [ | |
{ | |
"SpecialToken": { | |
"id": "UTT_BOUNDARY", | |
"type_id": 0 | |
} | |
}, | |
{ | |
"Sequence": { | |
"id": "A", | |
"type_id": 0 | |
} | |
} | |
], | |
"pair": [ | |
{ | |
"SpecialToken": { | |
"id": "UTT_BOUNDARY", | |
"type_id": 0 | |
} | |
}, | |
{ | |
"Sequence": { | |
"id": "A", | |
"type_id": 0 | |
} | |
}, | |
{ | |
"SpecialToken": { | |
"id": "UTT_BOUNDARY", | |
"type_id": 0 | |
} | |
}, | |
{ | |
"Sequence": { | |
"id": "B", | |
"type_id": 1 | |
} | |
} | |
], | |
"special_tokens": { | |
"UTT_BOUNDARY": { | |
"id": "UTT_BOUNDARY", | |
"ids": [ | |
3 | |
], | |
"tokens": [ | |
"UTT_BOUNDARY" | |
] | |
} | |
} | |
}, | |
"decoder": null, | |
"model": { | |
"type": "WordLevel", | |
"vocab": { | |
"UNK": 0, | |
"PAD": 1, | |
"WORD_BOUNDARY": 2, | |
"UTT_BOUNDARY": 3, | |
"j": 4, | |
"e̞": 5, | |
"s̪̻": 6, | |
"t̪̻": 7, | |
"u": 8, | |
"l": 9, | |
"o̞": 10, | |
"ʒ̺": 11, | |
"i": 12, | |
"ʋ": 13, | |
"d̪̻": 14, | |
"ä": 15, | |
"m": 16, | |
"n": 17, | |
"r": 18, | |
"k": 19, | |
"t̪̻s̪̻": 20, | |
"p": 21, | |
"ʃ̺": 22, | |
"x": 23, | |
"b": 24, | |
"ɡ": 25, | |
"t̻ʃ̻": 26, | |
"f": 27, | |
"z̪̻": 28, | |
"ɲ": 29, | |
"ʎ": 30, | |
"d̻ʒ̻": 31, | |
"y": 32, | |
"w": 33 | |
}, | |
"unk_token": "UNK" | |
} | |
} |