From 440d488ca801416c2517d1062a878cb10e3fe936 Mon Sep 17 00:00:00 2001 From: Marcus Greenwood Date: Thu, 7 Dec 2023 11:53:05 +0000 Subject: [PATCH 1/3] add relative movement plus diagonal movement for mouse keys - store the x_pos and y_pos so that move_to doesn't need to go back to home each time - add diagonal movement to mouse-keys for tiny speed boost when moving mouse --- routes/mouse.py | 69 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 19 deletions(-) diff --git a/routes/mouse.py b/routes/mouse.py index 02105a1..c1e76e0 100644 --- a/routes/mouse.py +++ b/routes/mouse.py @@ -12,6 +12,9 @@ # Set up mouse m = Mouse() +last_x_pos = -100 +last_y_pos = -100 + # # Config helper functions # def app_file_path(): # # Get the app filepath @@ -147,22 +150,32 @@ def api_mouse_keys_click(): return Response() def api_mouse_keys_move_to(): + global last_x_pos + global last_y_pos req = request.json - x_pos = req.get("x") - y_pos = req.get("y") + x_pos = int(req.get("x")) + y_pos = int(req.get("y")) + x_pos, y_pos = mapper.transformPoint(x_pos, y_pos, current_app.transformationMatrix) print("Mouse keys move: (%s, %s)" % (x_pos, y_pos)) - mouse_move_by(-500, -1000) - mouse_keys_move_by(0, 4) - mouse_keys_move_by(-10, 0) - mouse_keys_move_by(4, 0) - mouse_keys_move_by(0, -10) - mouse_keys_move_by(0, 4) - mouse_keys_move_by(x_pos, y_pos, transform = True) + if (last_x_pos > -100 and last_y_pos > -100): + mouse_keys_move_by(x_pos - last_x_pos, y_pos - last_y_pos) + else: + mouse_move_by(-1500, -2000) + mouse_keys_move_by(0, 4) + mouse_keys_move_by(-10, 0) + mouse_keys_move_by(4, 0) + mouse_keys_move_by(0, -10) + mouse_keys_move_by(0, 4) + mouse_keys_move_by(x_pos, y_pos) + + last_x_pos = x_pos + last_y_pos = y_pos + return Response(mimetype="application/json") def api_mouse_keys_move_home(): print("Mouse keys move home") - mouse_move_by(-500, -1000) + mouse_move_by(-1500, -2000) mouse_keys_move_by(0, 3) mouse_keys_move_by(-10, 0) mouse_keys_move_by(3, 0) @@ -193,22 +206,40 @@ def mouse_keys_move_by(x, y, transform = False): x_pos, y_pos = mapper.transformPoint(x, y, current_app.transformationMatrix) print("Transform point to: (%s, %s)" % (x_pos, y_pos)) - if x_pos < 0: - while x_pos < 0: + while x_pos != 0 or y_pos != 0: + + if x_pos < 0 and y_pos < 0: + send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP7) + x_pos += 1 + y_pos += 1 + + elif x_pos < 0 and y_pos > 0: + send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP1) + x_pos += 1 + y_pos -= 1 + + elif x_pos < 0: send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP4) x_pos += 1 - if x_pos > 0: - while x_pos > 0: + elif x_pos > 0 and y_pos < 0: + send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP9) + x_pos -= 1 + y_pos += 1 + + elif x_pos > 0 and y_pos > 0: + send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP3) + x_pos -= 1 + y_pos -= 1 + + elif x_pos > 0: send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP6) x_pos -= 1 - if y_pos < 0: - while y_pos < 0: + elif y_pos < 0: send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP8) y_pos += 1 - if y_pos > 0: - while y_pos > 0: + elif y_pos > 0: send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP2) - y_pos -= 1 \ No newline at end of file + y_pos -= 1 From 035cd6d6fe8b70ceb05ff2344d2070974fdcf16a Mon Sep 17 00:00:00 2001 From: Marcus Greenwood Date: Wed, 13 Dec 2023 11:36:03 +0000 Subject: [PATCH 2/3] changes --- config/mouse-config.json | 20 +++--- config/video-config.json | 10 +-- routes/keyboard.py | 9 +++ routes/mouse.py | 116 +++++++++++++++++++++++++++++------ server.py | 3 + video-driver/set-up-video.sh | 2 +- 6 files changed, 125 insertions(+), 35 deletions(-) diff --git a/config/mouse-config.json b/config/mouse-config.json index 0ed714b..3ef4dd4 100644 --- a/config/mouse-config.json +++ b/config/mouse-config.json @@ -1,23 +1,23 @@ { - "configured": false, + "configured": true, "point1": { "robot": { - "x": 0, - "y": 0 + "x": 10, + "y": 10 }, "screen": { - "x": 0, - "y": 0 + "x": 200, + "y": 194 } }, "point2": { "robot": { - "x": 0, - "y": 0 + "x": 10, + "y": 40 }, "screen": { - "x": 0, - "y": 0 + "x": 200, + "y": 612 } } -} \ No newline at end of file +} diff --git a/config/video-config.json b/config/video-config.json index d0a4171..7568d2b 100644 --- a/config/video-config.json +++ b/config/video-config.json @@ -1,11 +1,11 @@ { "vertical": { - "configured": false, - "timestamp": null, - "x": 0, + "configured": true, + "timestamp": "2023-12-01T12:54:06.653549+00:00", + "x": 710, "y": 0, - "w": 0, - "h": 0 + "w": 499, + "h": 1080 }, "horizontal": { "configured": false, diff --git a/routes/keyboard.py b/routes/keyboard.py index d724e5d..75fcfa1 100644 --- a/routes/keyboard.py +++ b/routes/keyboard.py @@ -11,4 +11,13 @@ def api_keyboard_type(): text = req.get("text") print("Keyboard type text: %s" % text) k.type(text) + return Response(mimetype="application/json") + +def api_keyboard_press(): + req = request.json + modifiers = req.get("modifiers") + key = req.get("key") + print("Keyboard press modifiers: %s" % modifiers) + print("Keyboard press key: %s" % key) + k.press(modifiers, key) return Response(mimetype="application/json") \ No newline at end of file diff --git a/routes/mouse.py b/routes/mouse.py index 02105a1..adaef49 100644 --- a/routes/mouse.py +++ b/routes/mouse.py @@ -12,6 +12,9 @@ # Set up mouse m = Mouse() +last_x_pos = -100 +last_y_pos = -100 + # # Config helper functions # def app_file_path(): # # Get the app filepath @@ -66,7 +69,46 @@ def api_mouse_move_by(): mouse_move_by(x_pos, y_pos) return Response(mimetype="application/json") +def api_mouse_move_to(): + global last_x_pos + global last_y_pos + req = request.json + x_pos = int(req.get("x")) + y_pos = int(req.get("y")) + #x_pos, y_pos = mapper.transformPoint(x_pos, y_pos, current_app.transformationMatrix) + print("Mouse move: (%s, %s)" % (x_pos, y_pos)) + if (last_x_pos > -100 and last_y_pos > -100): + mouse_move_by(x_pos - last_x_pos, y_pos - last_y_pos) + else: + mouse_move_by(-1500, -2000) + mouse_keys_move_by(0, 4) + mouse_keys_move_by(-10, 0) + mouse_keys_move_by(4, 0) + mouse_keys_move_by(0, -10) + mouse_keys_move_by(0, 4) + mouse_keys_move_by(x_pos, y_pos) + + last_x_pos = x_pos + last_y_pos = y_pos + + return Response(mimetype="application/json") + +def api_mouse_scroll(): + req = request.json + y_distance = int(req.get("y_distance")) + delta = 100 + if (y_distance < 0): + delta = -100 + + for x in range(abs(int(y_distance / 30))): + send_mouse_event('/dev/hidg1', 0x1, 0, 0, delta, 0) + + return Response(mimetype="application/json") + def api_mouse_move_home(): + global last_x_pos + global last_y_pos + print("Mouse move home") mouse_move_by(-1500, -2500) mouse_move_by(0, 40) @@ -74,6 +116,10 @@ def api_mouse_move_home(): mouse_move_by(40, 0) mouse_move_by(0, -100) mouse_move_by(0, 40) + + last_x_pos = 0 + last_y_pos = 0 + return Response(mimetype="application/json") def api_raw_mouse_drag_by(): @@ -147,27 +193,41 @@ def api_mouse_keys_click(): return Response() def api_mouse_keys_move_to(): + global last_x_pos + global last_y_pos req = request.json - x_pos = req.get("x") - y_pos = req.get("y") + x_pos = int(req.get("x")) + y_pos = int(req.get("y")) + x_pos, y_pos = mapper.transformPoint(x_pos, y_pos, current_app.transformationMatrix) print("Mouse keys move: (%s, %s)" % (x_pos, y_pos)) - mouse_move_by(-500, -1000) - mouse_keys_move_by(0, 4) - mouse_keys_move_by(-10, 0) - mouse_keys_move_by(4, 0) - mouse_keys_move_by(0, -10) - mouse_keys_move_by(0, 4) - mouse_keys_move_by(x_pos, y_pos, transform = True) + if (last_x_pos > -100 and last_y_pos > -100): + mouse_keys_move_by(x_pos - last_x_pos, y_pos - last_y_pos) + else: + mouse_move_by(-1500, -2000) + mouse_keys_move_by(0, 4) + mouse_keys_move_by(-10, 0) + mouse_keys_move_by(4, 0) + mouse_keys_move_by(0, -10) + mouse_keys_move_by(0, 4) + mouse_keys_move_by(x_pos, y_pos) + + last_x_pos = x_pos + last_y_pos = y_pos + return Response(mimetype="application/json") def api_mouse_keys_move_home(): print("Mouse keys move home") - mouse_move_by(-500, -1000) + mouse_move_by(-1500, -2000) mouse_keys_move_by(0, 3) mouse_keys_move_by(-10, 0) mouse_keys_move_by(3, 0) mouse_keys_move_by(0, -10) mouse_keys_move_by(0, 3) + global last_x_pos + global last_y_pos + last_x_pos = 0 + last_y_pos = 0 return Response(mimetype="application/json") def api_raw_mouse_keys_drag_by(): @@ -193,22 +253,40 @@ def mouse_keys_move_by(x, y, transform = False): x_pos, y_pos = mapper.transformPoint(x, y, current_app.transformationMatrix) print("Transform point to: (%s, %s)" % (x_pos, y_pos)) - if x_pos < 0: - while x_pos < 0: + while x_pos != 0 or y_pos != 0: + + if x_pos < 0 and y_pos < 0: + send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP7) + x_pos += 1 + y_pos += 1 + + elif x_pos < 0 and y_pos > 0: + send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP1) + x_pos += 1 + y_pos -= 1 + + elif x_pos < 0: send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP4) x_pos += 1 - if x_pos > 0: - while x_pos > 0: + elif x_pos > 0 and y_pos < 0: + send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP9) + x_pos -= 1 + y_pos += 1 + + elif x_pos > 0 and y_pos > 0: + send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP3) + x_pos -= 1 + y_pos -= 1 + + elif x_pos > 0: send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP6) x_pos -= 1 - if y_pos < 0: - while y_pos < 0: + elif y_pos < 0: send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP8) y_pos += 1 - if y_pos > 0: - while y_pos > 0: + elif y_pos > 0: send_keystroke('/dev/hidg0', 0, KeyCodes.KEY_KP2) - y_pos -= 1 \ No newline at end of file + y_pos -= 1 diff --git a/server.py b/server.py index 91c1bec..8af1da3 100644 --- a/server.py +++ b/server.py @@ -82,6 +82,8 @@ def api_ping(): app.add_url_rule('/api/mouse/up', view_func=mouse.api_mouse_up, methods=['POST']) app.add_url_rule('/api/mouse/click', view_func=mouse.api_mouse_click, methods=['POST']) app.add_url_rule('/api/mouse/move/by', view_func=mouse.api_mouse_move_by, methods=['POST']) +app.add_url_rule('/api/mouse/move/to', view_func=mouse.api_mouse_move_to, methods=['POST']) +app.add_url_rule('/api/mouse/scroll', view_func=mouse.api_mouse_scroll, methods=['POST']) app.add_url_rule('/api/mouse/move/home', view_func=mouse.api_mouse_move_home, methods=['POST']) app.add_url_rule('/api/raw/mouse/drag/by', view_func=mouse.api_raw_mouse_drag_by, methods=['POST']) app.add_url_rule('/api/raw/mouse/move/by', view_func=mouse.api_raw_mouse_move_by, methods=['POST']) @@ -98,6 +100,7 @@ def api_ping(): # Keyboard Routes app.add_url_rule('/api/keyboard/type', view_func=keyboard.api_keyboard_type, methods=['POST']) +app.add_url_rule('/api/keyboard/press', view_func=keyboard.api_keyboard_press, methods=['POST']) # Video Routes app.add_url_rule('/stream', view_func=video.stream) diff --git a/video-driver/set-up-video.sh b/video-driver/set-up-video.sh index bf4227a..18a3ed9 100755 --- a/video-driver/set-up-video.sh +++ b/video-driver/set-up-video.sh @@ -1,6 +1,6 @@ echo "1. Setting edid..." v4l2-ctl --set-edid=file=1080p30edid --fix-edid-checksums -sleep 4 +sleep 10 echo "2. Setting digital video timings..." # Success: From 69824e308c8719ca99869fafc700b431d39db024 Mon Sep 17 00:00:00 2001 From: Marcus Greenwood Date: Wed, 17 Jan 2024 19:35:42 +0000 Subject: [PATCH 3/3] add mouse routes --- routes/mouse.py | 19 +++++++++++++++++++ server.py | 5 ++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/routes/mouse.py b/routes/mouse.py index adaef49..b4c5678 100644 --- a/routes/mouse.py +++ b/routes/mouse.py @@ -38,6 +38,16 @@ # Mouse # ###################################################################### + +def debug_send_keys(): + req = request.json + path = req.get("path") + control_keys = 0 + hid_keycode = int(req.get("hid_keycode")) + release = bool(req.get("release")) + send_keystroke(path, control_keys, hid_keycode, release) + return Response() + def api_mouse_jiggle(): m.move_relative(10,0) time.sleep(.5) @@ -61,6 +71,15 @@ def api_mouse_click(): send_mouse_event('/dev/hidg1', 0x0, 0, 0, 0, 0) return Response() +def api_mouse_button(): + req = request.json + button = int(req.get("button")) + print("Mouse click button") + send_mouse_event('/dev/hidg1', button, 0, 0, 0, 0) + time.sleep(.05) + send_mouse_event('/dev/hidg1', 0x0, 0, 0, 0, 0) + return Response() + def api_mouse_move_by(): req = request.json x_pos = req.get("x") diff --git a/server.py b/server.py index 45c42ba..717a9e0 100644 --- a/server.py +++ b/server.py @@ -105,6 +105,7 @@ def api_ping(): app.add_url_rule('/api/mouse/down', view_func=mouse.api_mouse_down, methods=['POST']) app.add_url_rule('/api/mouse/up', view_func=mouse.api_mouse_up, methods=['POST']) app.add_url_rule('/api/mouse/click', view_func=mouse.api_mouse_click, methods=['POST']) +app.add_url_rule('/api/mouse/button', view_func=mouse.api_mouse_button, methods=['POST']) app.add_url_rule('/api/mouse/move/by', view_func=mouse.api_mouse_move_by, methods=['POST']) app.add_url_rule('/api/mouse/move/to', view_func=mouse.api_mouse_move_to, methods=['POST']) app.add_url_rule('/api/mouse/scroll', view_func=mouse.api_mouse_scroll, methods=['POST']) @@ -126,6 +127,9 @@ def api_ping(): app.add_url_rule('/api/keyboard/type', view_func=keyboard.api_keyboard_type, methods=['POST']) app.add_url_rule('/api/keyboard/press', view_func=keyboard.api_keyboard_press, methods=['POST']) +# Debug Routes +app.add_url_rule('/api/debug/send-keys', view_func=mouse.debug_send_keys, methods=['POST']) + # Video Routes if video_source == "hdmi": app.add_url_rule('/stream', view_func=video.stream) @@ -154,7 +158,6 @@ def api_ping(): app.add_url_rule('/raw/video-feed', view_func=video.raw_video_feed) app.add_url_rule('/api/config/video/camera', view_func=video.api_config_video, methods=['POST']) - if __name__ == '__main__': # Debug/Development app.run(debug=False, host="0.0.0.0", port="5000")