Upload folder using huggingface_hub
Browse files- tokenizer.json +13 -4
- vocab.json +0 -0
tokenizer.json
CHANGED
|
@@ -64,10 +64,19 @@
|
|
| 64 |
},
|
| 65 |
"post_processor": null,
|
| 66 |
"decoder": {
|
| 67 |
-
"type": "
|
| 68 |
-
"
|
| 69 |
-
|
| 70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
},
|
| 72 |
"model": {
|
| 73 |
"type": "BPE",
|
|
|
|
| 64 |
},
|
| 65 |
"post_processor": null,
|
| 66 |
"decoder": {
|
| 67 |
+
"type": "Sequence",
|
| 68 |
+
"decoders": [
|
| 69 |
+
{
|
| 70 |
+
"type": "BPEDecoder",
|
| 71 |
+
"suffix": "</w>"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"type": "ByteLevel",
|
| 75 |
+
"add_prefix_space": true,
|
| 76 |
+
"trim_offsets": true,
|
| 77 |
+
"use_regex": true
|
| 78 |
+
}
|
| 79 |
+
]
|
| 80 |
},
|
| 81 |
"model": {
|
| 82 |
"type": "BPE",
|
vocab.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|