Upload 5 files
Browse files- processing_kosmos2.py +5 -5
- tokenization_kosmos2.py +2 -2
- tokenization_kosmos2_fast.py +3 -3
processing_kosmos2.py
CHANGED
@@ -21,11 +21,11 @@ from typing import List, Optional, Tuple, Union
|
|
21 |
|
22 |
import numpy as np
|
23 |
|
24 |
-
from
|
25 |
-
from
|
26 |
-
from
|
27 |
-
from
|
28 |
-
from
|
29 |
|
30 |
|
31 |
if is_torch_available():
|
|
|
21 |
|
22 |
import numpy as np
|
23 |
|
24 |
+
from transformers.image_processing_utils import BatchFeature
|
25 |
+
from transformers.image_utils import ImageInput, is_batched
|
26 |
+
from transformers.processing_utils import ProcessorMixin
|
27 |
+
from transformers.tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy
|
28 |
+
from transformers.utils import TensorType, is_tf_available, is_torch_available
|
29 |
|
30 |
|
31 |
if is_torch_available():
|
tokenization_kosmos2.py
CHANGED
@@ -21,8 +21,8 @@ from typing import Any, Dict, List, Optional, Tuple
|
|
21 |
|
22 |
import sentencepiece as spm
|
23 |
|
24 |
-
from
|
25 |
-
from
|
26 |
|
27 |
|
28 |
logger = logging.get_logger(__name__)
|
|
|
21 |
|
22 |
import sentencepiece as spm
|
23 |
|
24 |
+
from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
|
25 |
+
from transformers.utils import logging
|
26 |
|
27 |
|
28 |
logger = logging.get_logger(__name__)
|
tokenization_kosmos2_fast.py
CHANGED
@@ -19,9 +19,9 @@ import os
|
|
19 |
from shutil import copyfile
|
20 |
from typing import List, Optional, Tuple
|
21 |
|
22 |
-
from
|
23 |
-
from
|
24 |
-
from
|
25 |
|
26 |
|
27 |
if is_sentencepiece_available():
|
|
|
19 |
from shutil import copyfile
|
20 |
from typing import List, Optional, Tuple
|
21 |
|
22 |
+
from transformers.tokenization_utils import AddedToken
|
23 |
+
from transformers.tokenization_utils_fast import PreTrainedTokenizerFast
|
24 |
+
from transformers.utils import is_sentencepiece_available, logging
|
25 |
|
26 |
|
27 |
if is_sentencepiece_available():
|