Skip to content

Commit cbc50d5

Browse files
mahbdmahbd
authored
Added windows local server API to help AI (#682)
* feat: add Windows UI automation inspection and background DOM monitoring endpoints * changed from * import to Specific import * improved windows ai chatbot inspection * fix special character app not supporting * Automatically fallback to gui if normal click failed * feat: enhance UI tree XML generation with ElementTree and improved structure * feat: add hotkey functionality to capture UI tree on demand
1 parent 00eac03 commit cbc50d5

3 files changed

Lines changed: 428 additions & 60 deletions

File tree

‎Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py‎

Lines changed: 81 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,22 @@
6666
# this needs to be here on top, otherwise will return error
6767
import clr, System
6868
dll_path = os.getcwd().split("Framework")[0] + "Framework" + os.sep + "windows_dll_files" + os.sep
69-
clr.AddReference(dll_path+"UIAutomationClient")
70-
clr.AddReference(dll_path+"UIAutomationTypes")
71-
clr.AddReference(dll_path+"UIAutomationProvider")
69+
clr.AddReference(dll_path + "UIAutomationClient")
70+
clr.AddReference(dll_path + "UIAutomationTypes")
71+
clr.AddReference(dll_path + "UIAutomationProvider")
7272
clr.AddReference("System.Windows.Forms")
7373

74-
from System.Windows.Automation import *
74+
from System.Windows.Automation import (
75+
AutomationElement,
76+
TreeScope,
77+
Condition,
78+
Automation,
79+
InvokePattern,
80+
ValuePattern,
81+
TogglePattern,
82+
SelectionItemPattern,
83+
ExpandCollapsePattern,
84+
)
7585
import pyautogui # Should be removed after we complete sequential actions
7686
import autoit # The likely method we'll use
7787

@@ -209,72 +219,83 @@ def Click_Element_None_Mouse(Element, Expand=True, Gui=False, offset: str | None
209219
pattern_name = Automation.PatternName(each)
210220
CommonUtil.ExecLog(sModuleInfo, "Pattern name attached to the current element is: %s " % pattern_name, 1)
211221

212-
# Expand and collapse actions
213-
if pattern_name == "ExpandCollapse":
214-
if Expand:
215-
# check to see if its expanded, if expanded, then do nothing... if not, expand it
216-
status = Element.GetCurrentPattern(
217-
ExpandCollapsePattern.Pattern
218-
).Current.ExpandCollapseState
219-
if status == 0:
220-
CommonUtil.ExecLog(sModuleInfo, "Expanding the item", 1)
221-
Element.GetCurrentPattern(
222+
try:
223+
# Expand and collapse actions
224+
if pattern_name == "ExpandCollapse":
225+
if Expand:
226+
# check to see if its expanded, if expanded, then do nothing... if not, expand it
227+
status = Element.GetCurrentPattern(
222228
ExpandCollapsePattern.Pattern
223-
).Expand()
224-
return "passed"
225-
elif status == 1:
226-
CommonUtil.ExecLog(sModuleInfo, "Already expanded", 1)
227-
return "passed"
228-
else:
229-
# check to see if its Collapsed, if Collapsed, then do nothing... if not, Collapse it
230-
status = Element.GetCurrentPattern(
231-
ExpandCollapsePattern.Pattern
232-
).Current.ExpandCollapseState
233-
if status == 1:
234-
CommonUtil.ExecLog(sModuleInfo, "Collapsing the item", 1)
235-
Element.GetCurrentPattern(
229+
).Current.ExpandCollapseState
230+
if status == 0:
231+
CommonUtil.ExecLog(sModuleInfo, "Expanding the item", 1)
232+
Element.GetCurrentPattern(
233+
ExpandCollapsePattern.Pattern
234+
).Expand()
235+
return "passed"
236+
elif status == 1:
237+
CommonUtil.ExecLog(sModuleInfo, "Already expanded", 1)
238+
return "passed"
239+
else:
240+
# check to see if its Collapsed, if Collapsed, then do nothing... if not, Collapse it
241+
status = Element.GetCurrentPattern(
236242
ExpandCollapsePattern.Pattern
237-
).Collapse()
238-
return "passed"
239-
elif status == 0:
240-
CommonUtil.ExecLog(sModuleInfo, "Already collapsed", 1)
241-
return "passed"
242-
# Invoking actions
243-
elif pattern_name == "Invoke":
244-
CommonUtil.ExecLog(sModuleInfo, "Invoking the object", 1)
245-
time.sleep(unnecessary_sleep)
246-
Element.GetCurrentPattern(InvokePattern.Pattern).Invoke()
247-
return "passed"
248-
# Selection of an item
249-
elif pattern_name == "SelectionItem":
250-
CommonUtil.ExecLog(sModuleInfo, "Selecting an item", 1)
251-
Element.GetCurrentPattern(SelectionItemPattern.Pattern).Select()
252-
time.sleep(unnecessary_sleep)
253-
return "passed"
254-
# Toggling action
255-
256-
elif pattern_name == "Toggle":
257-
CommonUtil.ExecLog(sModuleInfo, "Toggling an item", 1)
258-
Element.GetCurrentPattern(TogglePattern.Pattern).Toggle()
259-
time.sleep(unnecessary_sleep)
260-
return "passed"
261-
# if no patterns are found, then we do an actual mouse click
262-
else:
263-
# x = int (Element.Current.BoundingRectangle.X)
264-
# y = int (Element.Current.BoundingRectangle.Y)
243+
).Current.ExpandCollapseState
244+
if status == 1:
245+
CommonUtil.ExecLog(sModuleInfo, "Collapsing the item", 1)
246+
Element.GetCurrentPattern(
247+
ExpandCollapsePattern.Pattern
248+
).Collapse()
249+
return "passed"
250+
elif status == 0:
251+
CommonUtil.ExecLog(sModuleInfo, "Already collapsed", 1)
252+
return "passed"
253+
# Invoking actions
254+
elif pattern_name == "Invoke":
255+
CommonUtil.ExecLog(sModuleInfo, "Invoking the object", 1)
256+
time.sleep(unnecessary_sleep)
257+
Element.GetCurrentPattern(InvokePattern.Pattern).Invoke()
258+
return "passed"
259+
# Selection of an item
260+
elif pattern_name == "SelectionItem":
261+
CommonUtil.ExecLog(sModuleInfo, "Selecting an item", 1)
262+
Element.GetCurrentPattern(SelectionItemPattern.Pattern).Select()
263+
time.sleep(unnecessary_sleep)
264+
return "passed"
265+
# Toggling action
266+
elif pattern_name == "Toggle":
267+
CommonUtil.ExecLog(sModuleInfo, "Toggling an item", 1)
268+
Element.GetCurrentPattern(TogglePattern.Pattern).Toggle()
269+
time.sleep(unnecessary_sleep)
270+
return "passed"
271+
except Exception as e:
265272
CommonUtil.ExecLog(
266273
sModuleInfo,
267-
"We did not find any pattern for this object, so we will click by mouse with location",
268-
1,
274+
f"Normal click ({pattern_name}) failed or did nothing ({e}). Automatically using GUI click.",
275+
2,
269276
)
270-
x, y = get_coords(Element)
277+
x, y = get_coords(Element, offset)
271278
win32api.SetCursorPos((x, y))
272279
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN, x, y, 0, 0)
273280
time.sleep(0.1)
274281
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP, x, y, 0, 0)
275282
time.sleep(unnecessary_sleep)
276283
return "passed"
277284

285+
# if no patterns matched the standard ones, then we do an actual mouse click as fallback
286+
CommonUtil.ExecLog(
287+
sModuleInfo,
288+
"We did not find any suitable pattern for this object, so we will click by mouse with location",
289+
1,
290+
)
291+
x, y = get_coords(Element, offset)
292+
win32api.SetCursorPos((x, y))
293+
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN, x, y, 0, 0)
294+
time.sleep(0.1)
295+
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP, x, y, 0, 0)
296+
time.sleep(unnecessary_sleep)
297+
return "passed"
298+
278299
CommonUtil.ExecLog(sModuleInfo, "Unable to perform the action on the object", 3)
279300
return "zeuz_failed"
280301
except Exception:
@@ -309,7 +330,7 @@ def Check_uncheck(data_set):
309330
if command == "check" and is_selected == "On":
310331
CommonUtil.ExecLog(sModuleInfo, "The element is already checked so skipped it", 1)
311332
return "passed"
312-
elif command == "uncheck" and not is_selected:
333+
elif command == "uncheck" and is_selected == "Off":
313334
CommonUtil.ExecLog(sModuleInfo, "The element is already unchecked so skipped it", 1)
314335
return "passed"
315336
try:
@@ -2152,7 +2173,7 @@ def Run_Application(data_set):
21522173
#last_start_time = time.time()
21532174
autoit.send("^{ESC}")
21542175
time.sleep(keypress_interval)
2155-
autoit.send(Desktop_app)
2176+
autoit.send(Desktop_app, 1)
21562177
time.sleep(keypress_interval)
21572178
autoit.send("{ENTER}")
21582179
CommonUtil.ExecLog(sModuleInfo, "Successfully launched your app", 1)

‎server/main.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from server.mobile import router as mobile_router, start_ui_dump_uploads
1212
from server.mac import router as mac_router
1313
from server.linux import router as linux_router
14+
from server.windows import router as windows_router, upload_windows_ui_dump
1415
from server.installers import router as installers_router
1516
import asyncio
1617

@@ -43,12 +44,14 @@ def main() -> FastAPI:
4344
v1router.include_router(mobile_router)
4445
v1router.include_router(mac_router)
4546
v1router.include_router(linux_router)
47+
v1router.include_router(windows_router)
4648
v1router.include_router(installers_router)
4749
app = FastAPI()
4850

4951
@app.on_event("startup")
5052
async def _start_background_uploads():
5153
start_ui_dump_uploads()
54+
asyncio.create_task(upload_windows_ui_dump())
5255
app.include_router(v1router)
5356

5457
origins = [

0 commit comments

Comments
 (0)