GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored from https://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
bfbfa5a3044ec8d1312f3782c78ea3b9246bf667

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

onnxruntime-extensions/test/static_test

test/static_test/test_strings.cc

84lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation. All rights reserved.
2// Licensed under the MIT License.
3
4#include "gtest/gtest.h"
5#include "string_utils.h"
6#include "text/re2_strings/string_regex_split_re.hpp"
7#include "text/string_ecmaregex_split.hpp"
8
9TEST(strings, std_regex_test) {
10 std::regex regex("[\u2700-\u27bf\U0001f650-\U0001f67f\U0001f600-\U0001f64f\u2600-\u26ff"
11 "\U0001f300-\U0001f5ff\U0001f900-\U0001f9ff\U0001fa70-\U0001faff"
12 "\U0001f680-\U0001f6ff]");
13
14 std::string test =u8"abcde😀🔍🦑😁🔍🎉😂🤣";
15 auto result = std::regex_replace(test, regex, "");
16 std::cout << test << std::endl;
17 std::cout << result << std::endl;
18}
19
20
21TEST(strings, regex_split) {
22 std::string input = "hello world";
23 re2::RE2 reg("(\\s)");
24 re2::RE2 keep_reg("\\s");
25 std::vector<std::string_view> tokens;
26 std::vector<int64_t> begin_offsets;
27 std::vector<int64_t> end_offsets;
28 RegexSplitImpl(input, reg, true, keep_reg, tokens, begin_offsets, end_offsets);
29 std::vector<std::string_view> expected_tokens{"hello", " ", " ", "world"};
30 std::vector<int64_t> expected_begin_offsets{0, 5, 6, 7};
31 std::vector<int64_t> expected_end_offsets{5, 6, 7, 12};
32 EXPECT_EQ(expected_tokens, tokens);
33 EXPECT_EQ(expected_begin_offsets, begin_offsets);
34 EXPECT_EQ(expected_end_offsets, end_offsets);
35}
36
37TEST(strings, regex_split_skip) {
38 std::string input = "hello world";
39 re2::RE2 reg("(\\s)");
40 re2::RE2 keep_reg("");
41 std::vector<std::string_view> tokens;
42 std::vector<int64_t> begin_offsets;
43 std::vector<int64_t> end_offsets;
44 RegexSplitImpl(input, reg, true, keep_reg, tokens, begin_offsets, end_offsets);
45 std::vector<std::string_view> expected_tokens{"hello", "world"};
46 std::vector<int64_t> expected_begin_offsets{0, 6};
47 std::vector<int64_t> expected_end_offsets{5, 11};
48 EXPECT_EQ(expected_tokens, tokens);
49 EXPECT_EQ(expected_begin_offsets, begin_offsets);
50 EXPECT_EQ(expected_end_offsets, end_offsets);
51}
52
53TEST(strings, regex_split_no_matched) {
54 std::string input = "helloworld";
55 std::regex reg("(\\s)");
56 std::regex keep_reg("");
57 std::vector<std::string_view> tokens;
58 std::vector<int64_t> begin_offsets;
59 std::vector<int64_t> end_offsets;
60 ECMARegexSplitImpl(input, reg, true, keep_reg, tokens, begin_offsets, end_offsets);
61 std::vector<std::string_view> expected_tokens{"helloworld"};
62 std::vector<int64_t> expected_begin_offsets{0};
63 std::vector<int64_t> expected_end_offsets{10};
64 EXPECT_EQ(expected_tokens, tokens);
65 EXPECT_EQ(expected_begin_offsets, begin_offsets);
66 EXPECT_EQ(expected_end_offsets, end_offsets);
67}
68
69TEST(strings, regex_split_begin_end_delim) {
70 std::string input = " hello world ";
71 std::regex reg("(\\s)");
72 std::regex keep_reg("\\s");
73 std::vector<std::string_view> tokens;
74 std::vector<int64_t> begin_offsets;
75 std::vector<int64_t> end_offsets;
76 ECMARegexSplitImpl(input, reg, true, keep_reg, tokens, begin_offsets, end_offsets);
77 std::vector<std::string_view> expected_tokens{" ", "hello"," ", "world", " "};
78 std::vector<int64_t> expected_begin_offsets{0, 1, 6, 7, 12};
79 std::vector<int64_t> expected_end_offsets{1, 6, 7, 12, 13};
80 EXPECT_EQ(expected_tokens, tokens);
81 EXPECT_EQ(expected_begin_offsets, begin_offsets);
82 EXPECT_EQ(expected_end_offsets, end_offsets);
83}
84
85