GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/TypeAgent

Public

mirrored from https://github.com/microsoft/TypeAgentAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
339ef52802e9b4e5f63c5a059db17e182920ec36

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

TypeAgent/dotnet/autoShell

dotnet/autoShell/UIAutomation.cs

318lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4using System;
5using System.Collections.Generic;
6using System.Diagnostics;
7using System.Linq;
8using System.Runtime.InteropServices;
9using System.Text;
10using System.Threading.Tasks;
11
12namespace autoShell;
13
14/// <summary>
15/// This is a placeholder for UIAutomation related functionalities.
16/// </summary>
17/// <remarks>Only use this as a LAST resort for demo purposes only!</remarks>
18[Obsolete("UIAutomation is a last-resort method and should be avoided in production code.")]
19internal sealed class UIAutomation
20{
21 /// <summary>
22 /// Uses UI Automation to navigate the Settings app and set the text size.
23 /// </summary>
24 /// <param name="percentage">The text scaling percentage (100-225).</param>
25 internal static void SetTextSizeViaUIAutomation(int percentage)
26 {
27 // UI Automation Property IDs (from UIAutomationClient.h)
28 const int UIA_AutomationIdPropertyId = 30011;
29
30 // UI Automation Pattern IDs
31 const int UIA_RangeValuePatternId = 10003;
32
33 const int maxRetries = 10;
34 const int retryDelayMs = 500;
35
36 try
37 {
38 // Create UI Automation instance
39 var uiAutomation = new UIAutomationClient.CUIAutomation();
40 UIAutomationClient.IUIAutomationElement settingsWindow = null;
41
42 // Wait for Settings window to appear and get it via FindWindow
43 for (int i = 0; i < maxRetries; i++)
44 {
45 // Find the Settings window by enumerating top-level windows with "Settings" in the title
46 // UWP apps use ApplicationFrameWindow class
47 IntPtr hWnd = IntPtr.Zero;
48 while ((hWnd =
49 AutoShell.FindWindowEx(IntPtr.Zero, hWnd, "ApplicationFrameWindow", null)) != IntPtr.Zero)
50 {
51 StringBuilder windowTitle = new StringBuilder(256);
52 int hr = AutoShell.GetWindowText(hWnd, windowTitle, windowTitle.Capacity);
53 Debug.WriteLine(windowTitle + $"(hResult: {hr})");
54 if (windowTitle.ToString().Contains("Settings", StringComparison.OrdinalIgnoreCase))
55 {
56 // Get the automation element directly from the window handle
57 settingsWindow = uiAutomation.ElementFromHandle(hWnd);
58 break;
59 }
60 }
61
62 if (settingsWindow != null)
63 {
64 break;
65 }
66
67 System.Threading.Thread.Sleep(retryDelayMs);
68 }
69
70 if (settingsWindow == null)
71 {
72 AutoShell.LogWarning("Could not find Settings window.");
73 return;
74 }
75
76 Debug.WriteLine("Found Settings window via FindWindowEx");
77
78 // Wait a moment for the UI to fully load
79 System.Threading.Thread.Sleep(500);
80
81 // Find and click the "Text Size" navigation item
82 var textSizeNavItem = FindTextSizeNavigationItem(uiAutomation, settingsWindow);
83 if (textSizeNavItem != null)
84 {
85 Debug.WriteLine("Found Text Size navigation item, clicking...");
86 ClickElement(textSizeNavItem);
87 System.Threading.Thread.Sleep(500); // Wait for page to load
88 }
89 else
90 {
91 Debug.WriteLine("Text Size navigation item not found, may already be on the page");
92 }
93
94 // Find the text size slider
95 var sliderCondition = uiAutomation.CreatePropertyCondition(
96 UIA_AutomationIdPropertyId,
97 "SystemSettings_EaseOfAccess_Experience_TextScalingDesktop_Slider");
98
99 UIAutomationClient.IUIAutomationElement slider = null;
100 for (int i = 0; i < maxRetries; i++)
101 {
102 slider = settingsWindow.FindFirst(
103 UIAutomationClient.TreeScope.TreeScope_Descendants,
104 sliderCondition);
105
106 if (slider != null)
107 {
108 break;
109 }
110
111 System.Threading.Thread.Sleep(retryDelayMs);
112 }
113
114 if (slider == null)
115 {
116 AutoShell.LogWarning("Could not find text size slider.");
117 return;
118 }
119
120 Debug.WriteLine("Found text size slider");
121
122 // Set the slider value using RangeValue pattern
123 var rangeValuePattern = (UIAutomationClient.IUIAutomationRangeValuePattern)slider.GetCurrentPattern(
124 UIA_RangeValuePatternId);
125
126 if (rangeValuePattern != null)
127 {
128 Debug.WriteLine($"Setting slider value to {percentage}");
129 rangeValuePattern.SetValue(percentage);
130 }
131 else
132 {
133 AutoShell.LogWarning("Slider does not support RangeValue pattern.");
134 return;
135 }
136
137 // Wait a moment for the value to be applied
138 System.Threading.Thread.Sleep(300);
139
140 // Focus the slider and simulate a keyboard event to trigger the Apply button to become enabled
141 // The Apply button only enables when it detects actual user input on the slider
142 try
143 {
144 slider.SetFocus();
145 System.Threading.Thread.Sleep(100);
146
147 // Send a neutral key event (press and release a key that doesn't change the value)
148 // Using VK_DELETE followed by setting the value again ensures the UI registers the change
149 keybd_event(VK_DELETE, 0, 0, IntPtr.Zero);
150 keybd_event(VK_DELETE, 0, KEYEVENTF_KEYUP, IntPtr.Zero);
151 System.Threading.Thread.Sleep(100);
152
153 // Reset to the desired value in case the key changed it
154 rangeValuePattern.SetValue(percentage);
155 System.Threading.Thread.Sleep(200);
156 }
157 catch (Exception ex)
158 {
159 Debug.WriteLine($"Error simulating input on slider: {ex.Message}");
160 }
161
162 // Find and click the Apply button
163 var applyButtonCondition = uiAutomation.CreatePropertyCondition(
164 UIA_AutomationIdPropertyId,
165 "SystemSettings_EaseOfAccess_Experience_TextScalingDesktop_ButtonRemove");
166
167 UIAutomationClient.IUIAutomationElement applyButton = null;
168 for (int i = 0; i < maxRetries; i++)
169 {
170 applyButton = settingsWindow.FindFirst(
171 UIAutomationClient.TreeScope.TreeScope_Descendants,
172 applyButtonCondition);
173
174 if (applyButton != null)
175 {
176 break;
177 }
178
179 System.Threading.Thread.Sleep(retryDelayMs);
180 }
181
182 if (applyButton != null)
183 {
184 Debug.WriteLine("Found Apply button, clicking...");
185 ClickElement(applyButton);
186 Console.WriteLine($"Text size set to {percentage}%");
187 }
188 else
189 {
190 AutoShell.LogWarning("Could not find Apply button. The setting may need to be applied manually.");
191 }
192 }
193 catch (Exception ex)
194 {
195 AutoShell.LogError(ex);
196 }
197 }
198
199 /// <summary>
200 /// Finds the "Text Size" navigation item in the Settings window.
201 /// </summary>
202 static UIAutomationClient.IUIAutomationElement FindTextSizeNavigationItem(
203 UIAutomationClient.CUIAutomation uiAutomation,
204 UIAutomationClient.IUIAutomationElement settingsWindow)
205 {
206 // UI Automation Property IDs
207 const int UIA_NamePropertyId = 30005;
208 const int UIA_ControlTypePropertyId = 30003;
209 const int UIA_ListItemControlTypeId = 50007;
210
211 try
212 {
213 // Look for elements that contain "Text Size" in their name
214 var nameCondition = uiAutomation.CreatePropertyCondition(
215 UIA_NamePropertyId,
216 "Text size");
217
218 var textSizeElement = settingsWindow.FindFirst(
219 UIAutomationClient.TreeScope.TreeScope_Descendants,
220 nameCondition);
221
222 if (textSizeElement != null)
223 {
224 return textSizeElement;
225 }
226
227 // Alternative: search for ListItem or similar control containing "Text Size"
228 var listItemCondition = uiAutomation.CreatePropertyCondition(
229 UIA_ControlTypePropertyId,
230 UIA_ListItemControlTypeId);
231
232 var listItems = settingsWindow.FindAll(
233 UIAutomationClient.TreeScope.TreeScope_Descendants,
234 listItemCondition);
235
236 for (int i = 0; i < listItems.Length; i++)
237 {
238 var item = listItems.GetElement(i);
239 string name = item.CurrentName;
240 if (name != null && name.Contains("Text size", StringComparison.OrdinalIgnoreCase))
241 {
242 return item;
243 }
244 }
245 }
246 catch (Exception ex)
247 {
248 Debug.WriteLine($"Error finding Text Size navigation item: {ex.Message}");
249 }
250
251 return null;
252 }
253
254 /// <summary>
255 /// Clicks a UI Automation element using the Invoke pattern or simulated click.
256 /// </summary>
257 static void ClickElement(UIAutomationClient.IUIAutomationElement element)
258 {
259 // UI Automation Pattern IDs
260 const int UIA_InvokePatternId = 10000;
261 const int UIA_SelectionItemPatternId = 10010;
262
263 try
264 {
265 // Try using the Invoke pattern first
266 var invokePattern = (UIAutomationClient.IUIAutomationInvokePattern)element.GetCurrentPattern(
267 UIA_InvokePatternId);
268
269 if (invokePattern != null)
270 {
271 invokePattern.Invoke();
272 return;
273 }
274
275 // Try using the SelectionItem pattern
276 var selectionItemPattern = (UIAutomationClient.IUIAutomationSelectionItemPattern)element.GetCurrentPattern(
277 UIA_SelectionItemPatternId);
278
279 if (selectionItemPattern != null)
280 {
281 selectionItemPattern.Select();
282 return;
283 }
284
285 // Fall back to simulating a click at the element's center
286 var rect = element.CurrentBoundingRectangle;
287 int x = (rect.left + rect.right) / 2;
288 int y = (rect.top + rect.bottom) / 2;
289
290 // Move cursor and click
291 SetCursorPos(x, y);
292 mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, IntPtr.Zero);
293 mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, IntPtr.Zero);
294 }
295 catch (Exception ex)
296 {
297 Debug.WriteLine($"Error clicking element: {ex.Message}");
298 }
299 }
300
301 // Mouse event constants for simulated clicks
302 private const uint MOUSEEVENTF_LEFTDOWN = 0x0002;
303 private const uint MOUSEEVENTF_LEFTUP = 0x0004;
304
305 // Keyboard event constants
306 private const uint KEYEVENTF_KEYUP = 0x0002;
307 private const byte VK_DELETE = 0x2E;
308
309 [DllImport("user32.dll")]
310 private static extern bool SetCursorPos(int X, int Y);
311
312 [DllImport("user32.dll")]
313 private static extern void mouse_event(uint dwFlags, int dx, int dy, uint dwData, IntPtr dwExtraInfo);
314
315 [DllImport("user32.dll")]
316 private static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, IntPtr dwExtraInfo);
317
318}
319