GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored from https://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
sayanshaw/docs-automation

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

onnxruntime-extensions/base

base/ustring.h

106lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation. All rights reserved.
2// Licensed under the MIT License.
3#pragma once
4
5#include "ocos.h"
6#include <vector>
7#include <string_view>
8
9// ustring needs a new implementation, due to the std::codecvt deprecation.
10// Wrap u32string with ustring, in case we will use other implementation in the future
11class ustring : public std::u32string {
12 public:
13 ustring() = default;
14
15 explicit ustring(const char* str) { assign(FromUTF8(str)); }
16
17 explicit ustring(const std::string& str) { assign(FromUTF8(str)); }
18
19 explicit ustring(const std::string_view& str) { assign(FromUTF8(str)); }
20
21 explicit ustring(const char32_t* str) : std::u32string(str) {}
22
23 explicit ustring(const std::u32string_view& str) : std::u32string(str) {}
24
25 explicit operator std::string() const { return ToUTF8(*this); }
26
27 static size_t EncodeUTF8Char(char* buffer, char32_t utf8_char) {
28 if (utf8_char <= 0x7F) {
29 *buffer = static_cast<char>(utf8_char);
30 return 1;
31 } else if (utf8_char <= 0x7FF) {
32 buffer[1] = static_cast<char>(0x80 | (utf8_char & 0x3F));
33 utf8_char >>= 6;
34 buffer[0] = static_cast<char>(0xC0 | utf8_char);
35 return 2;
36 } else if (utf8_char <= 0xFFFF) {
37 buffer[2] = static_cast<char>(0x80 | (utf8_char & 0x3F));
38 utf8_char >>= 6;
39 buffer[1] = static_cast<char>(0x80 | (utf8_char & 0x3F));
40 utf8_char >>= 6;
41 buffer[0] = static_cast<char>(0xE0 | utf8_char);
42 return 3;
43 } else {
44 buffer[3] = static_cast<char>(0x80 | (utf8_char & 0x3F));
45 utf8_char >>= 6;
46 buffer[2] = static_cast<char>(0x80 | (utf8_char & 0x3F));
47 utf8_char >>= 6;
48 buffer[1] = static_cast<char>(0x80 | (utf8_char & 0x3F));
49 utf8_char >>= 6;
50 buffer[0] = static_cast<char>(0xF0 | utf8_char);
51 return 4;
52 }
53 }
54
55 static std::string EncodeUTF8Char(char32_t utf8_char) {
56 char utf8_buf[5]; // one extra space for zero
57 auto clen = EncodeUTF8Char(utf8_buf, utf8_char);
58 utf8_buf[clen] = 0;
59 return std::string(utf8_buf);
60 }
61
62 private:
63 using u32string = std::u32string;
64 static u32string FromUTF8(const std::string_view& utf8) {
65 u32string ucs32;
66 ucs32.reserve(utf8.length() / 2); // a rough estimation for less memory allocation.
67 for (size_t i = 0; i < utf8.size();) {
68 char32_t codepoint = 0;
69 if ((utf8[i] & 0x80) == 0) {
70 codepoint = utf8[i];
71 i++;
72 } else if ((utf8[i] & 0xE0) == 0xC0) {
73 codepoint = ((utf8[i] & 0x1F) << 6) | (utf8[i + 1] & 0x3F);
74 i += 2;
75 } else if ((utf8[i] & 0xF0) == 0xE0) {
76 codepoint = ((utf8[i] & 0x0F) << 12) | ((utf8[i + 1] & 0x3F) << 6) | (utf8[i + 2] & 0x3F);
77 i += 3;
78 } else {
79 codepoint = ((utf8[i] & 0x07) << 18) | ((utf8[i + 1] & 0x3F) << 12) | ((utf8[i + 2] & 0x3F) << 6) | (utf8[i + 3] & 0x3F);
80 i += 4;
81 }
82 ucs32.push_back(codepoint);
83 }
84 return ucs32;
85 }
86
87 static std::string ToUTF8(const u32string& ucs32) {
88 std::string utf8;
89 utf8.reserve(ucs32.length() * 4);
90 for (char32_t codepoint : ucs32) {
91 utf8 += EncodeUTF8Char(codepoint);
92 }
93
94 return utf8;
95 }
96};
97
98namespace std {
99template <>
100struct hash<ustring> {
101 size_t operator()(const ustring& __str) const noexcept {
102 hash<u32string> standard_hash;
103 return standard_hash(static_cast<u32string>(__str));
104 }
105};
106} // namespace std
107