add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Nov07_07-08-33_a3dfd813c39f/1636269050.325692/events.out.tfevents.1636269050.a3dfd813c39f.282.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3046d38393dab6055830c27463db45cd1c9def84a1aeb3bf92794bb622555a18
+size 4576

runs/Nov07_07-08-33_a3dfd813c39f/events.out.tfevents.1636269050.a3dfd813c39f.282.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9be2303e6869496d92c8169f603376ef2008818f0c376875253c3f12949b73c7
+size 4607

vocab.json

@@ -1 +1 @@
-{"ஜ": 0, "ோ": 1, "ஒ": 2, "க": 3, "'": 4, "ீ": 5, "ப": 6, "்": 7, "ச": 8, "ஈ": 9, "ூ": 10, "ே": 11, "ஔ": 12, "உ": 13, "ஷ": 14, "ை": 15, "ய": 16, "அ": 17, "ஏ": 18, "ஸ": 19, "எ": 20, "ண": 21, "ல": 22, "ஐ": 23, "ஃ": 24, "ு": 25, "ஹ": 26, "ஓ": 27, "ஆ": 28, "ம": 29, "ஞ": 30, "வ": 31, "ற": 32, "இ": 33, "ழ": 34, "ன": 35, "ெ": 36, "ௌ": 37, "ா": 38, "ங": 39, "ி": 40, "ஊ": 42, "ந": 43, "ள": 44, "ர": 45, "த": 46, "’": 47, "ட": 48, "ொ": 49, "|": 41, "[UNK]": 50, "[PAD]": 51}
+{"ம": 0, "ன": 1, "ி": 2, "எ": 3, "ொ": 4, "ட": 5, "வ": 6, "ஒ": 7, "ீ": 8, "ஊ": 9, "ஞ": 10, "'": 11, "ெ": 12, "ழ": 13, "ஏ": 14, "ச": 15, "ஐ": 16, "உ": 17, "ங": 18, "ு": 19, "ஈ": 20, "ஸ": 21, "ா": 22, "த": 23, "ண": 24, "ோ": 25, "ஆ": 26, "ஹ": 28, "்": 29, "ஷ": 30, "ஃ": 31, "ல": 32, "அ": 33, "க": 34, "ஜ": 35, "ள": 36, "ர": 37, "ந": 38, "ஓ": 39, "ை": 40, "ய": 41, "ஔ": 42, "ப": 43, "’": 44, "ூ": 45, "ற": 46, "ௌ": 47, "இ": 48, "ே": 49, "|": 27, "[UNK]": 50, "[PAD]": 51}