hermeschen-ezcon commited on
Commit
1d873f3
·
verified ·
1 Parent(s): 2cda4fb

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +0 -8
tokenizer_config.json CHANGED
@@ -8,18 +8,10 @@
8
  "clean_up_tokenization_spaces": false,
9
  "cls_token": "<|begin_of_sentence|>",
10
  "eos_token": "</s>",
11
- "extra_special_tokens": [
12
- "<|IMAGE_PLACEHOLDER|>",
13
- "<|image_pad|>",
14
- "<|IMAGE_START|>",
15
- "<|IMAGE_END|>",
16
- "<|video_pad|>"
17
- ],
18
  "is_local": true,
19
  "legacy": true,
20
  "mask_token": "<mask:1>",
21
  "model_max_length": 131072,
22
- "model_specific_special_tokens": {},
23
  "pad_token": "<unk>",
24
  "processor_class": "PaddleOCRVLProcessor",
25
  "sep_token": "<|end_of_sentence|>",
 
8
  "clean_up_tokenization_spaces": false,
9
  "cls_token": "<|begin_of_sentence|>",
10
  "eos_token": "</s>",
 
 
 
 
 
 
 
11
  "is_local": true,
12
  "legacy": true,
13
  "mask_token": "<mask:1>",
14
  "model_max_length": 131072,
 
15
  "pad_token": "<unk>",
16
  "processor_class": "PaddleOCRVLProcessor",
17
  "sep_token": "<|end_of_sentence|>",