GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/hve-core

Public

mirrored from https://github.com/microsoft/hve-coreAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
feat/1637-l4-tests

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

hve-core/scripts/evals/Modules

scripts/evals/Modules/retext-runner.mjs

261lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// SPDX-License-Identifier: MIT
3//
4// retext-runner.mjs
5//
6// Runs alex.js (inclusive-language linter) and retext-profanities against
7// stimulus prompt text supplied via a JSON manifest on stdin. Emits a JSON
8// report on stdout and exits with code 1 when any message is flagged.
9//
10// Manifest schema:
11// [{ "spec": "<rel-path>", "stimulus": "<name>", "text": "<prompt>" }, ...]
12//
13// Report schema:
14// { "results": [ { spec, stimulus, source, messages: [{rule, message, line, column}] } ] }
15
16import { stdin as input, stdout as output, stderr } from 'node:process';
17import { text as alexText } from 'alex';
18import { unified } from 'unified';
19import retextEnglish from 'retext-english';
20import retextProfanities from 'retext-profanities';
21import retextStringify from 'retext-stringify';
22
23// Phrase-aware allowlist keyed by rule ID. When a rule fires, the ±60-char
24// window around the match is tested against each regex. A match suppresses
25// the message, so bare uses ("abuse") still flag while established technical
26// bigrams ("token abuse", "penetration test") pass through.
27const PHRASE_ALLOWLIST = {
28 execution: [
29 /\b(code|command|remote|arbitrary|script|query|task|job|pipeline|workflow|test|order|parallel|sequential|tool|function|program|process|step)[\s-]+execution\b/i,
30 /\bexecution\s+(context|environment|order|mode|model|engine|plan|policy|time|path|trace|step|flow|phase)\b/i,
31 ],
32 execute: [
33 /\b(can|may|will|to|cannot|must|shall|should|able\s+to|allowed\s+to|attempts?\s+to|tries\s+to)\s+execute\b/i,
34 /\bexecute\s+(the|a|an|this|that|code|commands?|scripts?|queries|query|tests?|workflows?|steps?|actions?|tools?)\b/i,
35 ],
36 executes: [/\bexecutes?\s+(the|a|an|in|on|when|once|after|before|with|via|inside|against)\b/i],
37 executed: [
38 /\b(is|was|been|gets?|being|are|were)\s+executed\b/i,
39 /\bexecuted\s+(by|in|on|when|with|via|against|successfully|inside)\b/i,
40 ],
41 attack: [/\battack\s+(surface|vector|tree|chain|path|pattern|scenario|model|simulation|graph)\b/i],
42 attacks: [/\b(injection|replay|phishing|brute[- ]?force|dos|ddos|mitm|csrf|xss|sql|prompt|side[- ]?channel|timing|downgrade|impersonation)\s+attacks?\b/i],
43 failure: [
44 /\bfailure\s+(modes?|points?|rate|domain|recovery|handling|scenarios?|injection)\b/i,
45 /\b(single\s+point\s+of|point\s+of|build|test|deployment|pipeline|silent|graceful|hardware|network|system|cascading|partial|validation)\s+failure\b/i,
46 ],
47 failures: [/\b(test|build|pipeline|deployment|validation|cascading|silent|partial|transient)\s+failures\b/i],
48 failed: [
49 /\b(test|build|step|job|request|attempt|validation|check|deployment|login|authentication)\s+failed\b/i,
50 /\bfailed\s+(to|with|because|due|tests?|requests?|attempts?|jobs?|builds?|logins?)\b/i,
51 ],
52 abuse: [
53 /\b(token|privilege|api|rate[- ]?limit|resource|trust|process|permission|credential|service|account|session|workflow|pipeline|cache|memory|tool|prompt|model|context|chain|insider|lateral|optimization|reservation|scalper|automated)[\s-]+abuse\b/i,
54 /\bbusiness\s+logic\s+abuse\b/i,
55 /\babuse\s+(of\s+)?(tokens?|privileges?|apis?|rate[- ]?limits?|resources?|trust|processes?|permissions?|credentials?|services?|accounts?|sessions?|tools?)\b/i,
56 /\babuse\s+(prevention|scenarios?|the\s+\w+)\b/i,
57 /\b(to|of|for|against|from|by|contain|prevent|reduce|stop|mitigate|deter|resist|combat|enable|enables|enabling|allow|allows|allowing|cause|causes|causing|make|makes|making|trigger|triggers|triggering|detect|detects|detecting|report|reports|reporting|monitor|monitors|monitoring|investigate|investigates|investigating|susceptible\s+to|vulnerable\s+to|prone\s+to|subject\s+to|protect\s+against|guard\s+against|safeguard\s+against|defend\s+against)\s+abuse\b/i,
58 ],
59 abuses: [/\babuses\s+(permissions?|trust|tokens?|credentials?|rate[- ]?limits?|access)\b/i],
60 penetration: [
61 /\bpen(etration)?[- ]?test(ing|er|ers|s)?\b/i,
62 /\b(renewable|market|grid|water|gas|oil|broadband|internet|solar|wind)\s+penetration\b/i,
63 ],
64 invalid: [
65 /\binvalid\s+(input|token|argument|arguments?|request|signature|state|format|payload|configuration|key|certificate|hash|json|yaml|xml|url|uri|path|response|reference|operation|character|syntax|schema|type|value|parameter|option|credential|claim|header|message|field|entry|record|file|user|session|cursor)\b/i,
66 ],
67 'host-hostess': [
68 /\b(http|https|host|virtual|bastion|jump|docker|container|kubernetes|kube|vm|web|database|build|target|source|remote|local|origin|destination|build|runner|agent)\s+host\b/i,
69 /\bhost\s+(header|name|names|file|key|machine|machines|os|process|system|environment|configuration|address|port|computer)\b/i,
70 ],
71 'hostesses-hosts': [
72 /\bhosts?\s+(file|header|key|name|configuration|environment)\b/i,
73 /\b(virtual|build|target|remote|local|allowed|trusted|known)\s+hosts\b/i,
74 ],
75 white: [
76 /\bwhite[- ]?list(ed|ing|s)?\b/i,
77 /\bwhite[- ]?paper\b/i,
78 /\bwhite[- ]?box\b/i,
79 /\bwhite[- ]?hat\b/i,
80 /\bwhite[- ]?spac(e|es|ing)\b/i,
81 /\bwhite\s+(background|text|fill|colou?r)\b/i,
82 /\bblack[- ]?and[- ]?white\b/i,
83 /\bplain\s+white\b/i,
84 /\bWHITE\b/,
85 /\btext\s+(is\s+)?white\b/i,
86 ],
87 premature: [/\bpremature\s+(optimization|optimisation|return|exit|termination|closure|abort|completion)\b/i],
88 remains: [/\bremains?\s+(valid|stable|consistent|the\s+same|unchanged|active|available|open|closed|empty|in|at|on)\b/i],
89 color: [
90 /\b(syntax|terminal|theme|background|foreground|text|font|highlight|border|accent|primary|secondary|css|hex|rgb|rgba|hsl|ansi)\s+colou?rs?\b/i,
91 /\bcolou?rs?\s+(scheme|theme|palette|code|codes|map|space|picker|wheel|value|values)\b/i,
92 ],
93 colors: [/\b(syntax|terminal|theme|background|foreground|text|font|highlight|border|accent|primary|secondary|css|hex|rgb|rgba|hsl|ansi)\s+colou?rs\b/i],
94 period: [/\b(time|grace|trial|retention|warm[- ]?up|cool[- ]?down|warranty|notice|review|incubation|sampling|polling|wait|sleep|sla)\s+period\b/i],
95 periods: [/\b(time|grace|trial|retention|sampling|polling)\s+periods\b/i],
96 uk: [/\b(uk|u\.k\.)\s+(government|gov|english|spelling|date|locale|user|users|region|usage)\b/i],
97 australian: [/\baustralian\s+(english|spelling|locale|date|user|users|region)\b/i],
98 cracks: [
99 /\b(password|hash|encryption|crypto|code)\s+crack(s|ing|ed|er)?\b/i,
100 /\bcrack(s|ing|ed)?\s+(the\s+)?(password|hash|code|encryption|cipher)\b/i,
101 /\bfall(s|ing)?\s+through\s+the\s+cracks\b/i,
102 ],
103 crack: [
104 /\b(password|hash|encryption|crypto|code)\s+crack(s|ing|ed|er)?\b/i,
105 /\b(guess|brute[- ]?force)\s+or\s+crack\b/i,
106 ],
107 threeway: [/\bthree[- ]?way\s+(handshake|merge|join|sync|comparison|matching|reconciliation)\b/i],
108 black: [
109 /\bblack[- ]?box\b/i,
110 /\bblack[- ]?list(ed|ing|s)?\b/i,
111 /\bblack[- ]?hat\b/i,
112 /\bBlack\s+(formatter|format|compatible)\b/,
113 /\bBlack\s+Friday\b/i,
114 ],
115 trap: [
116 /\b(trap[- ]?door|trap\s+handler|trap\s+event|debug\s+trap|signal\s+trap|stack\s+trap|error\s+trap)\b/i,
117 /\b(common|easy|classic|usual|interface|design|prompt|mockup|fidelity)\W+trap\b/i,
118 /\b(keyboard|focus|tab|mouse|character[- ]key)\s+trap\b/i,
119 /\bno\s+keyboard\s+trap\b/i,
120 /\bfocus[- ]trap\b/i,
121 /\btrap\s+(focus|pages?|the\s+user|zone)\b/i,
122 /\b(break|order)\s+or\s+trap\b/i,
123 /\bmaintenance\s+trap\b/i,
124 /\|\s*Trap\s*\|/,
125 ],
126 traps: [
127 /\b(keyboard|focus|tab|mouse|character[- ]key)\s+traps\b/i,
128 /\btraps\s+(keyboard\s+)?focus\b/i,
129 ],
130 devils: [/\bdevil['\u2019]s\s+advocate\b/i],
131 god: [/\bgod[- ]?(object|class(es)?|mode|method(s)?|node(s)?)\b/i],
132 drug: [
133 /\bdrug[- ]drug\b/i,
134 /\bdrug\s+(data|dosage|administration|trial|interaction|safety|protocol|delivery|works?|for|provide|application|development|discovery|candidate|product|pricing|class|target|efficacy|approval|label|repurposing|substance|molecule|pipeline|formulation|company|maker|manufacturer|incentives?)\b/i,
135 /\b(new|orphan|investigational|approved|existing|better|active|single[- ]pathway|generic|branded|prescription|specialty|biologic|small[- ]molecule|a|the|novel|psychiatric|oncology|cardiovascular)\s+drugs?\b/i,
136 ],
137 drugs: [
138 /\b(new|orphan|investigational|approved|existing|better|active|single[- ]pathway|generic|branded|prescription|specialty|biologic|small[- ]molecule|the|novel|psychiatric|oncology|cardiovascular|of)\s+drugs\b/i,
139 /\bdrugs\s+(already|exist|and|or|are|were|that|which|for|in|to)\b/i,
140 ],
141 sexual: [/\bsexual\s+(dysfunction|health|function|activity|behaviou?r|orientation|wellbeing|well[- ]being|side[- ]effects?)\b/i],
142 gross: [/\bgross\s+(margin|profit|revenue|sales|income|weight|domestic|product|value|amount|booking|bookings)\b/i],
143 fu: [/\bxin\s+fu\b/i],
144 ass: [/\bAS['\u2019]s\b/],
145};
146
147const CONTEXT_RADIUS = 60;
148
149function messageOffsets(message) {
150 const place = message.place ?? message.position;
151 const start = place?.start?.offset;
152 const end = place?.end?.offset ?? start;
153 return start == null ? null : { start, end };
154}
155
156function isAllowedByPhrase(message, text) {
157 const patterns = PHRASE_ALLOWLIST[message.ruleId];
158 if (!patterns || patterns.length === 0) {
159 return false;
160 }
161 const offsets = messageOffsets(message);
162 if (!offsets) {
163 return false;
164 }
165 const windowStart = Math.max(0, offsets.start - CONTEXT_RADIUS);
166 const windowEnd = Math.min(text.length, offsets.end + CONTEXT_RADIUS);
167 const window = text.slice(windowStart, windowEnd);
168 return patterns.some((re) => re.test(window));
169}
170
171async function readStdin() {
172 let data = '';
173 input.setEncoding('utf8');
174 for await (const chunk of input) {
175 data += chunk;
176 }
177 return data;
178}
179
180function normalizeMessage(message, source) {
181 return {
182 source,
183 rule: message.ruleId ?? message.source ?? source,
184 message: message.reason ?? String(message),
185 line: message.line ?? null,
186 column: message.column ?? null,
187 };
188}
189
190async function runAlex(text) {
191 const vfile = alexText(text);
192 return (vfile.messages ?? [])
193 .filter((m) => !isAllowedByPhrase(m, text))
194 .map((m) => normalizeMessage(m, 'alex'));
195}
196
197const profanityProcessor = unified()
198 .use(retextEnglish)
199 .use(retextProfanities, { sureness: 1 })
200 .use(retextStringify);
201
202async function runProfanities(text) {
203 const file = await profanityProcessor.process(text);
204 return (file.messages ?? [])
205 .filter((m) => !isAllowedByPhrase(m, text))
206 .map((m) => normalizeMessage(m, 'retext-profanities'));
207}
208
209async function main() {
210 const raw = await readStdin();
211 if (!raw.trim()) {
212 output.write(JSON.stringify({ results: [] }) + '\n');
213 process.exitCode = 0;
214 return;
215 }
216
217 let manifest;
218 try {
219 manifest = JSON.parse(raw);
220 } catch (err) {
221 stderr.write(`retext-runner: failed to parse manifest JSON — ${err.message}\n`);
222 process.exitCode = 2;
223 return;
224 }
225
226 if (!Array.isArray(manifest)) {
227 stderr.write('retext-runner: manifest must be a JSON array\n');
228 process.exitCode = 2;
229 return;
230 }
231
232 const results = [];
233 let flagged = 0;
234
235 for (const item of manifest) {
236 const spec = item?.spec ?? '<unknown>';
237 const stimulus = item?.stimulus ?? '<unknown>';
238 const text = typeof item?.text === 'string' ? item.text : '';
239 if (!text.trim()) {
240 continue;
241 }
242
243 const [alexMessages, profMessages] = await Promise.all([
244 runAlex(text),
245 runProfanities(text),
246 ]);
247 const messages = [...alexMessages, ...profMessages];
248 if (messages.length > 0) {
249 flagged += messages.length;
250 results.push({ spec, stimulus, messages });
251 }
252 }
253
254 output.write(JSON.stringify({ results }) + '\n');
255 process.exitCode = flagged > 0 ? 1 : 0;
256}
257
258main().catch((err) => {
259 stderr.write(`retext-runner: unexpected error — ${err.stack ?? err.message}\n`);
260 process.exitCode = 2;
261});
262