Fix/window event race condition

openadapt/alembic/versions/d1b385041a20_add_a11y_event_remove_state_from_window_.py

@@ -0,0 +1,48 @@
+"""add_a11y_event_remove_state_from_window_event
+
+Revision ID: d1b385041a20
+Revises: bb25e889ad71
+Create Date: 2024-07-25 16:21:27.450372
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import sqlite
+import openadapt
+
+# revision identifiers, used by Alembic.
+revision = 'd1b385041a20'
+down_revision = 'bb25e889ad71'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('a11y_event',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True),
+    sa.Column('handle', sa.Integer(), nullable=True),
+    sa.Column('data', sa.JSON(), nullable=True),
+    sa.ForeignKeyConstraint(['handle', 'timestamp'], ['window_event.handle', 'window_event.timestamp'], name=op.f('fk_a11y_event_handle_window_event')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_a11y_event'))
+    )
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('handle', sa.Integer(), nullable=True))
+        batch_op.create_unique_constraint('uix_handle_timestamp', ['handle', 'timestamp'])
+        batch_op.drop_column('state')
+        batch_op.drop_column('window_id')
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('window_event', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('window_id', sa.VARCHAR(), nullable=True))
+        batch_op.add_column(sa.Column('state', sqlite.JSON(), nullable=True))
+        batch_op.drop_constraint('uix_handle_timestamp', type_='unique')
+        batch_op.drop_column('handle')
+
+    op.drop_table('a11y_event')
+    # ### end Alembic commands ###

openadapt/app/dashboard/api/recordings.py

@@ -11,6 +11,7 @@
 from openadapt.models import Recording
 from openadapt.plotting import display_event
 from openadapt.utils import image2utf8, row2dict
+from openadapt.visualize import dict2html
 
 
 class RecordingsAPI:
@@ -69,6 +70,38 @@ def recording_detail_route(self) -> None:
         @self.app.websocket("/{recording_id}")
         async def get_recording_detail(websocket: WebSocket, recording_id: int) -> None:
             """Get a specific recording and its action events."""
+
+            def extract_a11y_texts_value(
+                data: dict | list, target_key: str, target_class_name: str
+            ) -> list:
+                """Recursively extracts values from a nested dictionary.
+
+                Args:
+                    data (dict or list): The nested dict/list to search within.
+                    target_key (str): The key for which the values should be extracted.
+                    target_class_name (str): The value of the "friendly_class_name" key
+                        that must match for the target_key.
+
+                Returns:
+                    list: A list of values corresponding to the target_key where the
+                        "friendly_class_name" matches the target_class_name.
+                """
+                results = []
+
+                def recursive_extract(d: dict | list) -> None:
+                    if isinstance(d, dict):
+                        if d.get("friendly_class_name") == target_class_name:
+                            if target_key in d:
+                                results.append(d[target_key])
+                        for key, value in d.items():
+                            recursive_extract(value)
+                    elif isinstance(d, list):
+                        for item in d:
+                            recursive_extract(item)
+
+                recursive_extract(data)
+                return results
+
             await websocket.accept()
             session = crud.get_new_session(read_only=True)
             recording = crud.get_recording_by_id(session, recording_id)
@@ -112,6 +145,8 @@ def convert_to_str(event_dict: dict) -> dict:
 
             for action_event in action_events:
                 event_dict = row2dict(action_event)
+                a11y_dict = row2dict(action_event.window_event.a11y_event)
+                event_dict["a11y_data"] = dict2html(a11y_dict)
                 try:
                     image = display_event(action_event)
                     width, height = image.size

openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx

@@ -5,6 +5,7 @@ import { ActionEvent as ActionEventType } from '@/types/action-event'
 import { Accordion, Box, Grid, Image, Table } from '@mantine/core'
 import { useHover } from '@mantine/hooks';
 import { RemoveActionEvent } from './RemoveActionEvent';
+import './style.css'; 
 
 type Props = {
     event: ActionEventType;
@@ -41,7 +42,7 @@ export const ActionEvent = ({
     let content = (
         <Grid align='center'>
             <Grid.Col span={8}>
-                <Table w={400} withTableBorder withColumnBorders my={20} className='border-2 border-gray-300 border-solid'>
+                <Table withTableBorder withColumnBorders my={20} className='border-2 border-gray-300 border-solid'>
                     <Table.Tbody>
                         {typeof event.id === 'number' && (
                             <TableRowWithBorder>
@@ -136,6 +137,26 @@ export const ActionEvent = ({
                     </Table.Tbody>
                 </Table>
             </Grid.Col>
+            {event.a11y_data && (
+                <Grid.Col span={12}>
+                        <Accordion className="w-full">
+                            <Accordion.Item value="a11y_data">
+                                <Accordion.Control>
+                                    <div className="text-blue-500 w-full cursor-pointer">
+                                        Accessibility Data
+                                    </div>
+                                </Accordion.Control>
+                                <Accordion.Panel>
+                                    <Table  withTableBorder withColumnBorders my={20} className="border-2 border-gray-300 border-solid w-full">
+                                        <Table.Tbody>
+                                            <div className="table-nested border-2 border-gray-300 border-solid w-full" dangerouslySetInnerHTML={{ __html: event.a11y_data }} />
+                                        </Table.Tbody>
+                                    </Table>
+                                </Accordion.Panel>
+                            </Accordion.Item>
+                        </Accordion>   
+                </Grid.Col>
+            )}
             <Grid.Col span={4}>
                 <RemoveActionEvent event={event} />
             </Grid.Col>

openadapt/app/dashboard/components/ActionEvent/style.css

@@ -0,0 +1,20 @@
+.table-nested table {
+  border-collapse: collapse;
+  border: 1px solid #d1d5db;
+}
+
+.table-nested th,
+.table-nested td {
+  border: 1px solid #d1d5db;
+  padding: 8px; 
+  text-align: left;
+}
+
+.table-nested th {
+  background-color: #f9fafb;
+  font-weight: bold;
+}
+
+.table-nested td {
+  background-color: #ffffff;
+}

openadapt/app/dashboard/types/action-event.ts

@@ -27,6 +27,7 @@ export type ActionEvent = {
     dimensions?: { width: number, height: number };
     children?: ActionEvent[];
     words?: string[];
+    a11y_data: string;
     isComputed?: boolean;
     isOriginal?: boolean;
 }

openadapt/config.py

@@ -133,7 +133,7 @@ class SegmentationAdapter(str, Enum):
     OPENAI_MODEL_NAME: str = "gpt-3.5-turbo"
 
     # Record and replay
-    RECORD_WINDOW_DATA: bool = True
+    RECORD_A11Y_DATA: bool = True
     RECORD_READ_ACTIVE_ELEMENT_STATE: bool = False
     RECORD_VIDEO: bool
     RECORD_AUDIO: bool

openadapt/db/crud.py

@@ -27,6 +27,7 @@
     Screenshot,
     ScrubbedRecording,
     WindowEvent,
+    A11yEvent,
     copy_sa_instance,
 )
 from openadapt.privacy.base import ScrubbingProvider
@@ -262,6 +263,25 @@ def insert_recording(session: SaSession, recording_data: dict) -> Recording:
     return db_obj
 
 
+def insert_a11y_event(
+    session: SaSession,
+    event_data: dict,
+) -> None:
+    """Insert an a11y event into the database.
+
+    Args:
+        session (sa.orm.Session): The database session.
+        event_data (dict): The data of the event
+    """
+    handle = event_data["handle"]
+    a11y_data = event_data["a11y_data"]
+    timestamp = event_data["timestamp"]
+    a11y_event = A11yEvent(timestamp=timestamp, handle=handle, data=a11y_data)
+
+    session.add(a11y_event)
+    session.commit()
+
+
 def delete_recording(session: SaSession, recording: Recording) -> None:
     """Remove the recording from the db.
 
@@ -590,6 +610,35 @@ def get_window_events(
     )
 
 
+def get_a11y_events(
+    session: SaSession,
+    recording: Recording,
+) -> list[A11yEvent]:
+    """Get accessibility events for a given recording.
+
+    Args:
+        session (SaSession): The SQLAlchemy session.
+        recording (Recording): The recording object.
+
+    Returns:
+        list[A11yEvent]: A list of accessibility events for the recording.
+    """
+    return (
+        session.query(A11yEvent)
+        .join(
+            WindowEvent,
+            (A11yEvent.handle == WindowEvent.handle)
+            & (A11yEvent.timestamp == WindowEvent.timestamp),
+        )
+        .filter(WindowEvent.recording_id == recording.id)
+        .options(
+            joinedload(A11yEvent.window_event).joinedload(WindowEvent.recording),
+        )
+        .order_by(A11yEvent.timestamp)
+        .all()
+    )
+
+
 def disable_action_event(session: SaSession, event_id: int) -> None:
     """Disable an action event.