GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored fromhttps://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
a761fc59b7ace6392d8782fbb2a74888353ecf65

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

onnxruntime-extensions/base

base/string_utils.cc

194lines · modecode

RawDownload
Latest commit unavailable.
1#ifdef ENABLE_TF_STRING
2#include "farmhash.h"
3#endif
4
5#include "string_utils.h"
6
7std::vector<std::string_view> SplitString(const std::string_view& str, const std::string_view& seps, bool remove_empty_entries) {
8 std::vector<std::string_view> result;
9 std::string ::size_type pre_pos = 0;
10
11 //TODO: bug fix
12 while (true) {
13 auto next_pos = str.find_first_of(seps, pre_pos);
14
15 if (next_pos == std::string::npos) {
16 auto sub_str = str.substr(pre_pos, next_pos);
17 // sub_str is empty means the last sep reach the end of string
18 if (!sub_str.empty()) {
19 result.push_back(sub_str);
20 }
21
22 break;
23 }
24
25 if (pre_pos != next_pos || !remove_empty_entries) {
26 auto sub_str = str.substr(pre_pos, next_pos - pre_pos);
27 result.push_back(sub_str);
28 }
29
30 pre_pos = next_pos + 1;
31 }
32
33 return result;
34}
35
36bool IsCJK(char32_t c) {
37 return (c >= 0x4E00 && c <= 0x9FFF)
38 || (c >= 0x3400 && c <= 0x4DBF)
39 || (c >= 0x20000 && c <= 0x2A6DF)
40 || (c >= 0x2A700 && c <= 0x2B73F)
41 || (c >= 0x2B740 && c <= 0x2B81F)
42 || (c >= 0x2B820 && c <= 0x2CEAF)
43 || (c >= 0xF900 && c <= 0xFAFF)
44 || (c >= 0x2F800 && c <= 0x2FA1F);
45}
46
47// Generated by tools/generate_unicode_category_table.py
48bool IsSpace(char32_t c) {
49 if (c == 13||c == 32||c == 160||c == 8239||c == 8287||c == 12288) {
50 return true;
51 }
52
53 if ((c >= 9 && c <= 10)||(c >= 8192 && c <= 8202)) {
54 return true;
55 }
56
57 return false;
58}
59
60// Generated by tools/generate_unicode_category_table.py
61bool IsPunct(char32_t c) {
62 if (c == 161||c == 167||c == 171||c == 187||c == 191||c == 894||c == 903||c == 12336||c == 12349) {
63 return true;
64 }
65
66 if ((c >= 33 && c <= 47)||(c >= 58 && c <= 64)||(c >= 91 && c <= 96)||(c >= 123 && c <= 126)
67 ||(c >= 182 && c <= 183)||(c >= 8208 && c <= 8231)||(c >= 8240 && c <= 8259)
68 || (c >= 8261 && c <= 8273)||(c >= 8275 && c <= 8286)||(c >= 12289 && c <= 12291)
69 ||(c >= 12296 && c <= 12305)||(c >= 12308 && c <= 12319)) {
70 return true;
71 }
72
73 return false;
74}
75
76// Generated by tools/generate_unicode_category_table.py
77bool IsControl(char32_t c) {
78 if (c == 173||c == 907||c == 909||c == 930||c == 11930||c == 173790||c == 195102
79 ) {
80 return true;
81 }
82
83 if ((c >= 0 && c <= 8)||(c >= 11 && c <= 12)||(c >= 14 && c <= 31)||(c >= 128 && c <= 159)
84 ||(c >= 888 && c <= 889)||(c >= 896 && c <= 899)||(c >= 8203 && c <= 8207)
85 ||(c >= 8234 && c <= 8238)||(c >= 8288 && c <= 8302)||(c >= 12020 && c <= 12030)
86 ||(c >= 40957 && c <= 40958)||(c >= 64110 && c <= 64111)||(c >= 64218 && c <= 64254)
87 ||(c >= 177973 && c <= 177982)||(c >= 178206 && c <= 178207)||(c >= 183970 && c <= 183982)) {
88 return true;
89 }
90
91 return false;
92}
93
94bool IsAccent(char32_t c)
95{
96 // only support part of accent
97 // [TODO] support more accent
98 return c >= 0x300 && c <= 0x36F;
99}
100
101// only support latin now
102char32_t ToLower(char32_t c) {
103 if ((c >= 'A') && (c <= 'Z')) {
104 return c + 'a' - 'A';
105 }
106
107 if ((c >= U'À' && (c <= U'Þ'))) {
108 return c + U'à' - U'À';
109 }
110
111 return c;
112}
113
114char32_t StripAccent(char32_t c)
115{
116 // "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
117 const char32_t * tr = U"AAAAAAÆCEEEEIIIIÐNOOOOO×ØUUUUYÞßaaaaaaæceeeeiiiiðnooooo÷øuuuuyþy";
118 if (c < 192 || c > 255) {
119 return c;
120}
121
122 return tr[c - 192];
123}
124
125#ifdef ENABLE_TF_STRING
126// Source: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/hash.cc#L28
127static inline uint64_t ByteAs64(char c) { return static_cast<uint64_t>(c) & 0xff; }
128
129// Source: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/raw_coding.h#L41
130uint64_t DecodeFixed32(const char* ptr) {
131 return ((static_cast<uint64_t>(static_cast<unsigned char>(ptr[0]))) |
132 (static_cast<uint64_t>(static_cast<unsigned char>(ptr[1])) << 8) |
133 (static_cast<uint64_t>(static_cast<unsigned char>(ptr[2])) << 16) |
134 (static_cast<uint64_t>(static_cast<unsigned char>(ptr[3])) << 24));
135}
136
137// Source: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/raw_coding.h#L55
138static uint64_t DecodeFixed64(const char* ptr) {
139 uint64_t lo = DecodeFixed32(ptr);
140 uint64_t hi = DecodeFixed32(ptr + 4);
141 return (hi << 32) | lo;
142}
143
144// Source: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/hash.cc#L79
145uint64_t Hash64(const char* data, size_t n, uint64_t seed) {
146 const uint64_t m = 0xc6a4a7935bd1e995;
147 const int r = 47;
148
149 uint64_t h = seed ^ (n * m);
150
151 while (n >= 8) {
152 uint64_t k = DecodeFixed64(data);
153 data += 8;
154 n -= 8;
155
156 k *= m;
157 k ^= k >> r;
158 k *= m;
159
160 h ^= k;
161 h *= m;
162 }
163
164 switch (n) {
165 case 7:
166 h ^= ByteAs64(data[6]) << 48;
167 case 6:
168 h ^= ByteAs64(data[5]) << 40;
169 case 5:
170 h ^= ByteAs64(data[4]) << 32;
171 case 4:
172 h ^= ByteAs64(data[3]) << 24;
173 case 3:
174 h ^= ByteAs64(data[2]) << 16;
175 case 2:
176 h ^= ByteAs64(data[1]) << 8;
177 default: // case 1: make some code analyzer be happier.
178 h ^= ByteAs64(data[0]);
179 h *= m;
180 }
181
182 h ^= h >> r;
183 h *= m;
184 h ^= h >> r;
185
186 return h;
187}
188
189uint64_t Hash64Fast(const char* data, size_t n) {
190 return static_cast<int64_t>(util::Fingerprint64(data, n));
191}
192
193
194#endif // ENABLE_TF_STRING
195