Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/window event race condition #842

Open
wants to merge 43 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
4f68487
add joinedload to eagerly load related entities
Animesh404 Jul 3, 2024
85d0100
change get_active_window_data to return empty dict when state is none
Animesh404 Jul 4, 2024
1feea90
add eager as parameter in _get
Animesh404 Jul 4, 2024
418fdef
chore: try pygetwindows
Animesh404 Jul 5, 2024
3410e32
window-event performance test
Animesh404 Jul 8, 2024
b0a66ed
remove unwanted changes from other files
Animesh404 Jul 8, 2024
7ad454b
Merge branch 'main' into fix/WindowEvent-race-condition
Animesh404 Jul 8, 2024
51986f7
fix: add new A11yEvent model
Animesh404 Jul 16, 2024
8b0937d
Merge branch 'main' into fix/WindowEvent-race-condition
Animesh404 Jul 16, 2024
bba8101
remove unnecessary comment
Animesh404 Jul 16, 2024
f770251
Merge branch 'fix/WindowEvent-race-condition' of https://github.com/A…
Animesh404 Jul 16, 2024
a9ecc15
fix: change WindowEvent Model
Animesh404 Jul 16, 2024
ed43695
fixing crud and record.py for a11yevents
Animesh404 Jul 16, 2024
1543390
remove unnecessary test file as we are not using pygetwindow
Animesh404 Jul 16, 2024
28fe26d
fix: fixing data removal logic in WindowEvent model
Animesh404 Jul 17, 2024
8dabc63
add handle and remove window_id
Animesh404 Jul 18, 2024
bbe0c47
chore: db revision
Animesh404 Jul 18, 2024
488bf4b
fix: remove state from window_event and add a11y_counter
Animesh404 Jul 21, 2024
e89710c
remove state from window_event
Animesh404 Jul 22, 2024
574e133
fix: add a11y_event in visualize
Animesh404 Jul 23, 2024
51db67d
Merge branch 'main' into fix/WindowEvent-race-condition
Animesh404 Jul 23, 2024
6e60017
fix: replay error fix and black error fix
Animesh404 Jul 23, 2024
950c1a0
Merge branch 'main' of https://github.com/OpenAdaptAI/OpenAdapt into …
Animesh404 Jul 23, 2024
8b125e4
fix: formatting
Animesh404 Jul 23, 2024
4841c50
Merge branch 'fix/WindowEvent-race-condition' of https://github.com/A…
Animesh404 Jul 23, 2024
31c270f
fix: flake8 error
Animesh404 Jul 23, 2024
32a42ec
Update models.py
abrichr Jul 24, 2024
27e8c75
Update record.py: add a11y_event_reader
abrichr Jul 24, 2024
db6b05a
fix: add read_a11y_data in config
Animesh404 Jul 25, 2024
b275f35
conflict resolve
Animesh404 Jul 25, 2024
6623bb9
resolve conflicts
Animesh404 Jul 25, 2024
597f6c0
remove unnecessry comments and items
Animesh404 Jul 25, 2024
403750a
remove duplicate values from merge resolution
Animesh404 Jul 25, 2024
1148967
rename RECORD_WINDOW_DATA to RECORD_A11Y_DATA and state to a11y_data
Animesh404 Jul 25, 2024
85e23bb
fix: change to_prompt_dict in WindowEvent
Animesh404 Jul 25, 2024
5bfb833
resolve conflict
Animesh404 Jul 25, 2024
d402968
change to_prompt_dict in WindowEvent model
Animesh404 Jul 25, 2024
4147c1c
Merge branch 'main' into fix/WindowEvent-race-condition
Animesh404 Jul 25, 2024
25b830a
update to_prompt_dict
Animesh404 Jul 25, 2024
bf04f51
Merge branch 'fix/WindowEvent-race-condition' of https://github.com/A…
Animesh404 Jul 25, 2024
8c0d2e6
add a11y_data in dashboard
Animesh404 Jul 26, 2024
78ae338
Merge branch 'main' into fix/WindowEvent-race-condition
Animesh404 Jul 26, 2024
3b0ada0
use pygetwindow for window and pywinauto for a11y and display a11y on…
Animesh404 Jul 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion openadapt/window/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def get_active_window_data(
dict or None: A dictionary containing information about the active window,
or None if the state is not available.
"""
state = get_active_window_state(include_window_data)
state = get_active_window_state_pgw(include_window_data)
if not state:
return {}
title = state["title"]
Expand Down Expand Up @@ -67,6 +67,20 @@ def get_active_window_state(read_window_data: bool) -> dict | None:
return None


def get_active_window_state_pgw(read_window_data: bool) -> dict | None:
Animesh404 marked this conversation as resolved.
Show resolved Hide resolved
"""Get the state of the active window using pygetwindow.

Returns:
dict or None: A dictionary containing the state of the active window,
or None if the state is not available.
"""
try:
return impl.get_active_window_state_pgw(read_window_data)
except Exception as exc:
logger.warning(f"{exc=}")
return None


def get_active_element_state(x: int, y: int) -> dict | None:
"""Get the state of the active element at the specified coordinates.

Expand Down
80 changes: 80 additions & 0 deletions openadapt/window/_windows.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,63 @@
from pprint import pprint
import pygetwindow as gw
import pickle
import time

from loguru import logger
import pywinauto


def get_active_window_state_pgw(read_window_data: bool) -> dict:
Animesh404 marked this conversation as resolved.
Show resolved Hide resolved
try:
active_window = gw.getActiveWindow()
if not active_window:
raise RuntimeError("No active window found.")
Animesh404 marked this conversation as resolved.
Show resolved Hide resolved
except RuntimeError as e:
logger.warning(e)
return {}

meta = {
"title": active_window.title,
"left": active_window.left,
"top": active_window.top,
"width": active_window.width,
"height": active_window.height,
"rectangle": {
Animesh404 marked this conversation as resolved.
Show resolved Hide resolved
"left": active_window.left,
"top": active_window.top,
"right": active_window.right,
"bottom": active_window.bottom,
"width": active_window.width,
"height": active_window.height,
},
"control_id": active_window._hWnd,
}

if read_window_data:
data = get_element_properties_pgw(active_window)
else:
data = {}

state = {
"title": meta["title"],
"left": meta["left"],
"top": meta["top"],
"width": meta["width"],
"height": meta["height"],
"meta": meta,
"data": data,
"window_id": meta["control_id"],
}

try:
pickle.dumps(state)
Animesh404 marked this conversation as resolved.
Show resolved Hide resolved
except Exception as exc:
logger.warning(f"{exc=}")
state.pop("data")

return state


def get_active_window_state(read_window_data: bool) -> dict:
"""Get the state of the active window.

Expand Down Expand Up @@ -149,6 +201,34 @@ def dictify_rect(rect: pywinauto.win32structures.RECT) -> dict:
return rect_dict


def get_element_properties_pgw(window: gw.Window) -> dict:
"""Simulates retrieving properties of a window element and its children.

Args:
window: An instance of a pygetwindow Window.

Returns:
dict: A nested dictionary containing the properties of the window.
The dictionary includes a "children" key, which will be an empty list
since pygetwindow does not support child elements retrieval.
"""
properties = {
"title": window.title,
"left": window.left,
"top": window.top,
"width": window.width,
"height": window.height,
"rectangle": {
"left": window.left,
"top": window.top,
"width": window.width,
"height": window.height,
},
"children": [], # pygetwindow does not support children
Animesh404 marked this conversation as resolved.
Show resolved Hide resolved
Animesh404 marked this conversation as resolved.
Show resolved Hide resolved
}
return properties


def get_properties(element: pywinauto.application.WindowSpecification) -> dict:
"""Retrieves specific writable properties of an element.

Expand Down
89 changes: 89 additions & 0 deletions tests/openadapt/test_window_state.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import pytest
import time
from openadapt import window


def test_get_active_window_state():

state = window.get_active_window_state(True)

# Ensure state is a dictionary
assert isinstance(state, dict), "State should be a dictionary"

# Check for required keys
required_keys = {
"title",
"left",
"top",
"width",
"height",
"meta",
"data",
"window_id",
}
for key in required_keys:
assert key in state, f"Missing key in state: {key}"

# Check the type of each key
assert isinstance(state["title"], str), "Title should be a string"
assert isinstance(state["left"], int), "Left should be an integer"
assert isinstance(state["top"], int), "Top should be an integer"
assert isinstance(state["width"], int), "Width should be an integer"
assert isinstance(state["height"], int), "Height should be an integer"
assert isinstance(state["meta"], dict), "Meta should be a dictionary"
assert isinstance(state["data"], dict), "Data should be a dictionary"
assert isinstance(state["window_id"], int), "Window ID should be an integer"


def test_get_active_window_state_pgw():
state = window.get_active_window_state_pgw(True)

# Ensure state is a dictionary
assert isinstance(state, dict), "State should be a dictionary"

# Check for required keys
required_keys = {
"title",
"left",
"top",
"width",
"height",
"meta",
"data",
"window_id",
}
for key in required_keys:
assert key in state, f"Missing key in state: {key}"

# Check the type of each key
assert isinstance(state["title"], str), "Title should be a string"
assert isinstance(state["left"], int), "Left should be an integer"
assert isinstance(state["top"], int), "Top should be an integer"
assert isinstance(state["width"], int), "Width should be an integer"
assert isinstance(state["height"], int), "Height should be an integer"
assert isinstance(state["meta"], dict), "Meta should be a dictionary"
assert isinstance(state["data"], dict), "Data should be a dictionary"
assert isinstance(state["window_id"], int), "Window ID should be an integer"


def test_execution_time():
# Measure execution time for pywinauto-based function
start_time = time.perf_counter()
window.get_active_window_state(True)
pywinauto_duration = time.perf_counter() - start_time

# Measure execution time for pygetwindow-based function
start_time = time.perf_counter()
window.get_active_window_state_pgw(True)
pygetwindow_duration = time.perf_counter() - start_time

print(f"pygetwindow duration: {pygetwindow_duration:.10f}")
print(f"pywinauto duration: {pywinauto_duration:.10f}")

assert (
pygetwindow_duration < pywinauto_duration
), "pygetwindow should be faster than pywinauto"


if __name__ == "__main__":
pytest.main()
Loading