Skip to content

Added windows local server API to help AI #682

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mdshakib007 merged 7 commits into from
Apr 9, 2026
Merged

Added windows local server API to help AI #682

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
74ee057
feat: add Windows UI automation inspection and background DOM monitor…
mahbd Mar 30, 2026
e41e084
changed from * import to Specific import
mahbd Mar 31, 2026
aa83c44
improved windows ai chatbot inspection
mahbd Apr 1, 2026
b0f08be
fix special character app not supporting
mahbd Apr 2, 2026
3185ace
Automatically fallback to gui if normal click failed
mahbd Apr 2, 2026
178cb17
feat: enhance UI tree XML generation with ElementTree and improved st…
mahbd Apr 6, 2026
f317f72
feat: add hotkey functionality to capture UI tree on demand
mahbd Apr 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 81 additions & 60 deletions Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,22 @@
# this needs to be here on top, otherwise will return error
import clr, System
dll_path = os.getcwd().split("Framework")[0] + "Framework" + os.sep + "windows_dll_files" + os.sep
clr.AddReference(dll_path+"UIAutomationClient")
clr.AddReference(dll_path+"UIAutomationTypes")
clr.AddReference(dll_path+"UIAutomationProvider")
clr.AddReference(dll_path + "UIAutomationClient")
clr.AddReference(dll_path + "UIAutomationTypes")
clr.AddReference(dll_path + "UIAutomationProvider")
clr.AddReference("System.Windows.Forms")

from System.Windows.Automation import *
from System.Windows.Automation import (
AutomationElement,
TreeScope,
Condition,
Automation,
InvokePattern,
ValuePattern,
TogglePattern,
SelectionItemPattern,
ExpandCollapsePattern,
)
import pyautogui # Should be removed after we complete sequential actions
import autoit # The likely method we'll use

Expand Down Expand Up @@ -209,72 +219,83 @@ def Click_Element_None_Mouse(Element, Expand=True, Gui=False, offset: str | None
pattern_name = Automation.PatternName(each)
CommonUtil.ExecLog(sModuleInfo, "Pattern name attached to the current element is: %s " % pattern_name, 1)

# Expand and collapse actions
if pattern_name == "ExpandCollapse":
if Expand:
# check to see if its expanded, if expanded, then do nothing... if not, expand it
status = Element.GetCurrentPattern(
ExpandCollapsePattern.Pattern
).Current.ExpandCollapseState
if status == 0:
CommonUtil.ExecLog(sModuleInfo, "Expanding the item", 1)
Element.GetCurrentPattern(
try:
# Expand and collapse actions
if pattern_name == "ExpandCollapse":
if Expand:
# check to see if its expanded, if expanded, then do nothing... if not, expand it
status = Element.GetCurrentPattern(
ExpandCollapsePattern.Pattern
).Expand()
return "passed"
elif status == 1:
CommonUtil.ExecLog(sModuleInfo, "Already expanded", 1)
return "passed"
else:
# check to see if its Collapsed, if Collapsed, then do nothing... if not, Collapse it
status = Element.GetCurrentPattern(
ExpandCollapsePattern.Pattern
).Current.ExpandCollapseState
if status == 1:
CommonUtil.ExecLog(sModuleInfo, "Collapsing the item", 1)
Element.GetCurrentPattern(
).Current.ExpandCollapseState
if status == 0:
CommonUtil.ExecLog(sModuleInfo, "Expanding the item", 1)
Element.GetCurrentPattern(
ExpandCollapsePattern.Pattern
).Expand()
return "passed"
elif status == 1:
CommonUtil.ExecLog(sModuleInfo, "Already expanded", 1)
return "passed"
else:
# check to see if its Collapsed, if Collapsed, then do nothing... if not, Collapse it
status = Element.GetCurrentPattern(
ExpandCollapsePattern.Pattern
).Collapse()
return "passed"
elif status == 0:
CommonUtil.ExecLog(sModuleInfo, "Already collapsed", 1)
return "passed"
# Invoking actions
elif pattern_name == "Invoke":
CommonUtil.ExecLog(sModuleInfo, "Invoking the object", 1)
time.sleep(unnecessary_sleep)
Element.GetCurrentPattern(InvokePattern.Pattern).Invoke()
return "passed"
# Selection of an item
elif pattern_name == "SelectionItem":
CommonUtil.ExecLog(sModuleInfo, "Selecting an item", 1)
Element.GetCurrentPattern(SelectionItemPattern.Pattern).Select()
time.sleep(unnecessary_sleep)
return "passed"
# Toggling action

elif pattern_name == "Toggle":
CommonUtil.ExecLog(sModuleInfo, "Toggling an item", 1)
Element.GetCurrentPattern(TogglePattern.Pattern).Toggle()
time.sleep(unnecessary_sleep)
return "passed"
# if no patterns are found, then we do an actual mouse click
else:
# x = int (Element.Current.BoundingRectangle.X)
# y = int (Element.Current.BoundingRectangle.Y)
).Current.ExpandCollapseState
if status == 1:
CommonUtil.ExecLog(sModuleInfo, "Collapsing the item", 1)
Element.GetCurrentPattern(
ExpandCollapsePattern.Pattern
).Collapse()
return "passed"
elif status == 0:
CommonUtil.ExecLog(sModuleInfo, "Already collapsed", 1)
return "passed"
# Invoking actions
elif pattern_name == "Invoke":
CommonUtil.ExecLog(sModuleInfo, "Invoking the object", 1)
time.sleep(unnecessary_sleep)
Element.GetCurrentPattern(InvokePattern.Pattern).Invoke()
return "passed"
# Selection of an item
elif pattern_name == "SelectionItem":
CommonUtil.ExecLog(sModuleInfo, "Selecting an item", 1)
Element.GetCurrentPattern(SelectionItemPattern.Pattern).Select()
time.sleep(unnecessary_sleep)
return "passed"
# Toggling action
elif pattern_name == "Toggle":
CommonUtil.ExecLog(sModuleInfo, "Toggling an item", 1)
Element.GetCurrentPattern(TogglePattern.Pattern).Toggle()
time.sleep(unnecessary_sleep)
return "passed"
except Exception as e:
CommonUtil.ExecLog(
sModuleInfo,
"We did not find any pattern for this object, so we will click by mouse with location",
1,
f"Normal click ({pattern_name}) failed or did nothing ({e}). Automatically using GUI click.",
2,
)
x, y = get_coords(Element)
x, y = get_coords(Element, offset)
win32api.SetCursorPos((x, y))
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN, x, y, 0, 0)
time.sleep(0.1)
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP, x, y, 0, 0)
time.sleep(unnecessary_sleep)
return "passed"

# if no patterns matched the standard ones, then we do an actual mouse click as fallback
CommonUtil.ExecLog(
sModuleInfo,
"We did not find any suitable pattern for this object, so we will click by mouse with location",
1,
)
x, y = get_coords(Element, offset)
win32api.SetCursorPos((x, y))
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN, x, y, 0, 0)
time.sleep(0.1)
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP, x, y, 0, 0)
time.sleep(unnecessary_sleep)
return "passed"

CommonUtil.ExecLog(sModuleInfo, "Unable to perform the action on the object", 3)
return "zeuz_failed"
except Exception:
Expand Down Expand Up @@ -309,7 +330,7 @@ def Check_uncheck(data_set):
if command == "check" and is_selected == "On":
CommonUtil.ExecLog(sModuleInfo, "The element is already checked so skipped it", 1)
return "passed"
elif command == "uncheck" and not is_selected:
elif command == "uncheck" and is_selected == "Off":
CommonUtil.ExecLog(sModuleInfo, "The element is already unchecked so skipped it", 1)
return "passed"
try:
Expand Down Expand Up @@ -2152,7 +2173,7 @@ def Run_Application(data_set):
#last_start_time = time.time()
autoit.send("^{ESC}")
time.sleep(keypress_interval)
autoit.send(Desktop_app)
autoit.send(Desktop_app, 1)
time.sleep(keypress_interval)
autoit.send("{ENTER}")
CommonUtil.ExecLog(sModuleInfo, "Successfully launched your app", 1)
Expand Down
3 changes: 3 additions & 0 deletions server/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from server.mobile import router as mobile_router, start_ui_dump_uploads
from server.mac import router as mac_router
from server.linux import router as linux_router
from server.windows import router as windows_router, upload_windows_ui_dump
from server.installers import router as installers_router
import asyncio

Expand Down Expand Up @@ -43,12 +44,14 @@ def main() -> FastAPI:
v1router.include_router(mobile_router)
v1router.include_router(mac_router)
v1router.include_router(linux_router)
v1router.include_router(windows_router)
v1router.include_router(installers_router)
app = FastAPI()

@app.on_event("startup")
async def _start_background_uploads():
start_ui_dump_uploads()
asyncio.create_task(upload_windows_ui_dump())
app.include_router(v1router)

origins = [
Expand Down
Loading
Loading