GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored from https://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v0.7.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

onnxruntime-extensions/operators/tokenizer

operators/tokenizer/bert_tokenizer_decoder.hpp

49lines · modeblame

RawDownload
Latest commit unavailable.
97ec9507Mojimi4 years ago1// Copyright (c) Microsoft Corporation. All rights reserved.
2// Licensed under the MIT License.
3
4#pragma once
5
6#include "ocos.h"
7#include "ustring.h"
8#include "string_utils.h"
9#include "string_tensor.h"
10
11class BertTokenizerDecoder {
12public:
d8cdb8e0Mojimi4 years ago13BertTokenizerDecoder(std::string vocab, std::string unk_token, std::string sep_token, std::string pad_token,
5e44a7c3Scott McKay3 years ago14std::string cls_token, std::string mask_token, std::string suffix_indicator);
d8cdb8e0Mojimi4 years ago15std::string Decode(const std::vector<int64_t>& ids, bool skip_special_tokens, bool clean_up_tokenization_spaces);
97ec9507Mojimi4 years ago16
17private:
d8cdb8e0Mojimi4 years ago18std::string unk_token_;
5320af1eWenbing Li3 years ago19int32_t unk_token_id_ = -1;
20int32_t sep_token_id_ = -1;
21int32_t pad_token_id_ = -1;
22int32_t cls_token_id_ = -1;
23int32_t mask_token_id_ = -1;
d8cdb8e0Mojimi4 years ago24std::string suffix_indicator_;
25std::vector<std::string_view> vocab_;
26std::string raw_vocab_;
97ec9507Mojimi4 years ago27std::vector<bool> is_substr_;
cce66310Mojimi4 years ago28
d8cdb8e0Mojimi4 years ago29bool RemoveTokenizeSpace(int64_t pre_token_id, int64_t new_token_id);
97ec9507Mojimi4 years ago30};
31
32struct KernelBertTokenizerDecoder : BaseKernel {
5e44a7c3Scott McKay3 years ago33KernelBertTokenizerDecoder(const OrtApi& api, const OrtKernelInfo& info);
97ec9507Mojimi4 years ago34void Compute(OrtKernelContext* context);
5e44a7c3Scott McKay3 years ago35
97ec9507Mojimi4 years ago36private:
37std::shared_ptr<BertTokenizerDecoder> decoder_;
cce66310Mojimi4 years ago38bool use_indices_;
39bool skip_special_tokens_;
40bool clean_up_tokenization_spaces_;
97ec9507Mojimi4 years ago41};
42
c599b00dWenbing Li3 years ago43struct CustomOpBertTokenizerDecoder : OrtW::CustomOpBase<CustomOpBertTokenizerDecoder, KernelBertTokenizerDecoder> {
97ec9507Mojimi4 years ago44const char* GetName() const;
45size_t GetInputTypeCount() const;
46ONNXTensorElementDataType GetInputType(size_t index) const;
47size_t GetOutputTypeCount() const;
48ONNXTensorElementDataType GetOutputType(size_t index) const;
5e44a7c3Scott McKay3 years ago49};