GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored from https://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
rel-0.4

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

onnxruntime-extensions/operators/tokenizer

operators/tokenizer/bert_tokenizer_decoder.hpp

52lines · modeblame

RawDownload
Latest commit unavailable.
97ec9507Mojimi4 years ago1// Copyright (c) Microsoft Corporation. All rights reserved.
2// Licensed under the MIT License.
3
4#pragma once
5
6#include <unordered_map>
7#include <vector>
8#include "ocos.h"
9#include "ustring.h"
10#include "string_utils.h"
11#include "string_tensor.h"
12
13
14class BertTokenizerDecoder {
15public:
d8cdb8e0Mojimi4 years ago16BertTokenizerDecoder(std::string vocab, std::string unk_token, std::string sep_token, std::string pad_token,
17std::string cls_token,std::string mask_token,std::string suffix_indicator);
18std::string Decode(const std::vector<int64_t>& ids, bool skip_special_tokens, bool clean_up_tokenization_spaces);
97ec9507Mojimi4 years ago19
20private:
d8cdb8e0Mojimi4 years ago21std::string unk_token_;
97ec9507Mojimi4 years ago22int32_t unk_token_id_;
23int32_t sep_token_id_;
24int32_t pad_token_id_;
25int32_t cls_token_id_;
26int32_t mask_token_id_;
d8cdb8e0Mojimi4 years ago27std::string suffix_indicator_;
28std::vector<std::string_view> vocab_;
29std::string raw_vocab_;
97ec9507Mojimi4 years ago30std::vector<bool> is_substr_;
cce66310Mojimi4 years ago31
d8cdb8e0Mojimi4 years ago32bool RemoveTokenizeSpace(int64_t pre_token_id, int64_t new_token_id);
97ec9507Mojimi4 years ago33};
34
35struct KernelBertTokenizerDecoder : BaseKernel {
36KernelBertTokenizerDecoder(OrtApi api, const OrtKernelInfo* info);
37void Compute(OrtKernelContext* context);
38private:
39std::shared_ptr<BertTokenizerDecoder> decoder_;
cce66310Mojimi4 years ago40bool use_indices_;
41bool skip_special_tokens_;
42bool clean_up_tokenization_spaces_;
97ec9507Mojimi4 years ago43};
44
45struct CustomOpBertTokenizerDecoder : Ort::CustomOpBase<CustomOpBertTokenizerDecoder, KernelBertTokenizerDecoder> {
46void* CreateKernel(OrtApi api, const OrtKernelInfo* info) const;
47const char* GetName() const;
48size_t GetInputTypeCount() const;
49ONNXTensorElementDataType GetInputType(size_t index) const;
50size_t GetOutputTypeCount() const;
51ONNXTensorElementDataType GetOutputType(size_t index) const;
52};