GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored from https://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
rel-0.9

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

onnxruntime-extensions/onnxruntime_extensions

onnxruntime_extensions/_hf_cvt.py

219lines · modecode

RawDownload
Latest commit unavailable.
1# Copyright (c) Microsoft Corporation. All rights reserved.
2# Licensed under the MIT License. See License.txt in the project root for
3# license information.
4###############################################################################
5
6"""
7_hf_cvt.py: HuggingFace Tokenizer/Processor Converter
8"""
9
10import json
11import onnx
12from numpy import array as nparray
13from functools import partial
14from collections import namedtuple, OrderedDict
15
16from ._cuops import CustomOpConverter, SingleOpGraph
17from .util import read_file
18
19
20class HFTokenizerConverter(CustomOpConverter):
21 def __init__(self, tokenizer):
22 self.tokenizer = tokenizer
23
24 def bpe_tokenizer(self, **kwargs):
25 hf_gpt2_tokenizer = self.tokenizer
26
27 if type(self.tokenizer).__name__.endswith('Fast'):
28 raise ValueError('Please use the slow version of the tokenizer (ex: GPT2Tokenizer).')
29
30 attrs = {'vocab': json.dumps(
31 hf_gpt2_tokenizer.encoder, separators=(',', ':'))}
32 sorted_merges = {v_: k_ for k_, v_ in hf_gpt2_tokenizer.bpe_ranks.items()}
33 attrs['merges'] = '\n'.join("{} {}".format(
34 *sorted_merges[n_]) for n_ in range(len(sorted_merges)))
35 attrs.update(**kwargs)
36 return attrs
37
38 def bert_tokenizer(self, **kwargs):
39 hf_bert_tokenizer = self.tokenizer
40 # has to be sorted since the id of token was generated automatically.
41 ordered_vocab = OrderedDict(sorted(hf_bert_tokenizer.vocab.items(), key=lambda item: int(item[1])))
42 vocab = '\n'.join(ordered_vocab.keys())
43 attrs = dict(vocab=vocab)
44 init_kwargs = hf_bert_tokenizer.init_kwargs
45 attrs['do_lower_case'] = 1 if 'do_lower_case' in init_kwargs and init_kwargs.get('do_lower_case') else 0
46 attrs['strip_accents'] = 1 if 'strip_accents' in init_kwargs and init_kwargs.get('strip_accents') else 0
47 attrs.update(**kwargs)
48 return attrs
49
50 def bert_decoder(self, **kwargs):
51 hf_bert_tokenizer = self.tokenizer
52 attrs = {'vocab': json.dumps(
53 hf_bert_tokenizer.ids_to_tokens, separators=(',', ':'))}
54 attrs.update(**kwargs)
55 return attrs
56
57 def bpe_decoder(self, **kwargs):
58 decoder = self.tokenizer.decoder
59 id_vocab = "\n".join([decoder[_idx] for _idx in sorted(decoder)])
60 byte_decoder = self.tokenizer.byte_decoder
61 str_byte_decoder = "\n".join(["{}\t{}".format(
62 ord(_c), str(byte_decoder[_c])) for _c in byte_decoder])
63 all_special_ids = self.tokenizer.all_special_ids
64 added_tokens = self.tokenizer.added_tokens_decoder
65 str_all_special_ids = "\n".join([str(_id) for _id in all_special_ids])
66 str_added_tokens = "\n".join(
67 ["{}\t{}".format(str(_id), added_tokens[_id]) for _id in added_tokens])
68 kwargs.update({
69 "id_vocab": id_vocab,
70 "byte_decoder": str_byte_decoder,
71 "added_tokens": str_added_tokens,
72 "all_special_ids": str_all_special_ids,
73 "skip_special_tokens": kwargs.get("skip_special_tokens", False)
74 })
75 return kwargs
76
77 def clip_tokenizer(self, **kwargs):
78 hf_clip_tokenizer = self.tokenizer
79
80 if type(self.tokenizer).__name__.endswith('Fast'):
81 raise ValueError('Please use the slow version of the tokenizer (ex: CLIPTokenizer).')
82
83 attrs = {'vocab': json.dumps(
84 hf_clip_tokenizer.encoder, separators=(',', ':'))}
85 sorted_merges = {v_: k_ for k_,
86 v_ in hf_clip_tokenizer.bpe_ranks.items()}
87 attrs['merges'] = '\n'.join("{} {}".format(
88 *sorted_merges[n_]) for n_ in range(len(sorted_merges)))
89 attrs.update(**kwargs)
90 return attrs
91
92 def roberta_tokenizer(self, **kwargs):
93 hf_roberta_tokenizer = self.tokenizer
94
95 if type(self.tokenizer).__name__.endswith('Fast'):
96 raise ValueError('Please use the slow version of the tokenizer (ex: RobertaTokenizer).')
97
98 attrs = {'vocab': json.dumps(
99 hf_roberta_tokenizer.encoder, separators=(',', ':'))}
100 sorted_merges = {v_: k_ for k_,
101 v_ in hf_roberta_tokenizer.bpe_ranks.items()}
102 attrs['merges'] = '\n'.join("{} {}".format(
103 *sorted_merges[n_]) for n_ in range(len(sorted_merges)))
104 attrs.update(**kwargs)
105 return attrs
106
107 def spm_tokenizer(self, **kwargs):
108 attrs = {'model': read_file(self.tokenizer.vocab_file, 'rb')}
109 attrs.update(**kwargs)
110 return attrs
111
112 def spm_decoder(self, **kwargs):
113 attrs = {'model': read_file(self.tokenizer.vocab_file, 'rb')}
114 attrs.update(**kwargs)
115 return attrs
116
117
118TokenOpParam = namedtuple("TokenOpParam",
119 ["pre_op", "pre_attribute_cvt",
120 "post_op", "post_attribute_cvt",
121 "default_inputs"],
122 defaults=(None, None, None, None, None))
123
124# Some tokenizers can be added by this table
125# https://github.com/huggingface/transformers/blob/main/src/transformers/convert_slow_tokenizer.py#L1252
126# @formatter:off
127_PROCESSOR_DICT = {
128 "BertTokenizer": TokenOpParam('BertTokenizer', HFTokenizerConverter.bert_tokenizer,
129 'BertDecoder', HFTokenizerConverter.bpe_decoder, None),
130 "DistilBertTokenizer": TokenOpParam('BertTokenizer', HFTokenizerConverter.bert_tokenizer,
131 'BertDecoder', HFTokenizerConverter.bpe_decoder, None),
132 "GPT2Tokenizer": TokenOpParam('GPT2Tokenizer', HFTokenizerConverter.bpe_tokenizer,
133 'BpeDecoder', HFTokenizerConverter.bpe_decoder, None),
134 "CodeGenTokenizer": TokenOpParam('GPT2Tokenizer', HFTokenizerConverter.bpe_tokenizer,
135 'BpeDecoder', HFTokenizerConverter.bpe_decoder, None),
136 "CLIPTokenizer": TokenOpParam('CLIPTokenizer', HFTokenizerConverter.clip_tokenizer,
137 'BpeDecoder', HFTokenizerConverter.bpe_decoder, None),
138 "RobertaTokenizer": TokenOpParam('RobertaTokenizer', HFTokenizerConverter.roberta_tokenizer,
139 'BpeDecoder', HFTokenizerConverter.bpe_decoder, None),
140 "BartTokenizer": TokenOpParam('RobertaTokenizer', HFTokenizerConverter.roberta_tokenizer,
141 'BpeDecoder', HFTokenizerConverter.bpe_decoder, None),
142 "LayoutLMv3Tokenizer": TokenOpParam('RobertaTokenizer', HFTokenizerConverter.roberta_tokenizer,
143 'BpeDecoder', HFTokenizerConverter.bpe_decoder, None),
144 "LongformerTokenizer": TokenOpParam('RobertaTokenizer', HFTokenizerConverter.roberta_tokenizer,
145 'BpeDecoder', HFTokenizerConverter.bpe_decoder, None),
146 "LEDTokenizer": TokenOpParam('RobertaTokenizer', HFTokenizerConverter.roberta_tokenizer,
147 'BpeDecoder', HFTokenizerConverter.bpe_decoder, None),
148 "MvpTokenizer": TokenOpParam('RobertaTokenizer', HFTokenizerConverter.roberta_tokenizer,
149 'BpeDecoder', HFTokenizerConverter.bpe_decoder, None),
150 "T5Tokenizer": TokenOpParam('SentencepieceTokenizer', HFTokenizerConverter.spm_tokenizer,
151 'SentencepieceDecoder', HFTokenizerConverter.spm_decoder,
152 default_inputs={'add_eos': [True]}),
153 "LlamaTokenizer": TokenOpParam('SentencepieceTokenizer', HFTokenizerConverter.spm_tokenizer,
154 'SentencepieceDecoder', HFTokenizerConverter.spm_decoder,
155 default_inputs={'add_bos': [True]})
156}
157# @formatter:on
158
159
160class HFTokenizerOnnxGraph:
161
162 @staticmethod
163 def extract_cls_name(processor):
164 cls_name = processor if isinstance(processor, str) else type(processor).__name__
165 if cls_name.endswith("TokenizerFast"):
166 cls_name = cls_name[:-len("Fast")]
167 return cls_name
168
169 @classmethod
170 def is_supported(cls, processor):
171 cls_name = cls.extract_cls_name(processor)
172 return cls_name in _PROCESSOR_DICT
173
174 def __init__(self, processor, **kwargs):
175 cls_name = self.extract_cls_name(processor)
176 self.cvt_quadruple = _PROCESSOR_DICT[cls_name]
177 self.cvt_obj = HFTokenizerConverter(processor)
178
179 def pre_processing(self, **kwargs):
180 with_default_inputs = kwargs.pop("WITH_DEFAULT_INPUTS", True)
181 _cvt_op = self.cvt_quadruple.pre_op
182 _cvt_func = self.cvt_quadruple.pre_attribute_cvt
183 cvt = partial(_cvt_func, self.cvt_obj)
184 g = SingleOpGraph.build_graph(_cvt_op, cvt=cvt, **kwargs)
185 default_inputs = []
186 if with_default_inputs:
187 op_class = SingleOpGraph.get_op_class(_cvt_op)
188 default_inputs = op_class.input_default_values()
189 if default_inputs is None:
190 return g
191
192 # add default_inputs into initializers to simplify the model input
193 n_inputs = len(default_inputs)
194 if self.cvt_quadruple.default_inputs is not None:
195 default_inputs.update(self.cvt_quadruple.default_inputs)
196 if len(default_inputs) != n_inputs:
197 raise ValueError("Op: {} does have the inputs from its TokenOpParam.".format(_cvt_op))
198
199 new_initializers = []
200
201 for k, v in default_inputs.items():
202 input_value_info = next((i for i in g.input if i.name == k), None)
203 if input_value_info is None:
204 raise ValueError("The input {} is not found in the graph".format(k))
205
206 np_dtype = onnx.helper.tensor_dtype_to_np_dtype(input_value_info.type.tensor_type.elem_type)
207 value = nparray(v, np_dtype)
208 new_initializers.append(onnx.numpy_helper.from_array(value, k))
209 g.initializer.extend(new_initializers)
210 new_inputs = [i for i in g.input if i.name not in default_inputs]
211 g.ClearField("input")
212 g.input.extend(new_inputs)
213 return g
214
215 def post_processing(self, **kwargs):
216 _cvt_op = self.cvt_quadruple.post_op
217 _cvt_func = self.cvt_quadruple.post_attribute_cvt
218 cvt = partial(_cvt_func, self.cvt_obj)
219 return SingleOpGraph.build_graph(_cvt_op, cvt=cvt, **kwargs)
220