From dfec3d2bac1a31c830b641a58defb7b3ece1c9b9 Mon Sep 17 00:00:00 2001 From: pmohanj Date: Thu, 2 May 2024 18:24:41 +0530 Subject: [PATCH 1/2] Client sends key-repeat events to the server to indicate deliberate key held state to distinguish from key held state which might occur under some bad network conditions. The server releases those keys that are held due to bad network conditions by analysing the time interval of key-repeat events of the corresponding keys. --- addons/gst-web/src/app.js | 3 +++ addons/gst-web/src/input.js | 35 +++++++++++++++++++++++++++ addons/gst-web/src/util.js | 16 ++++++++++++ src/selkies_gstreamer/__main__.py | 1 + src/selkies_gstreamer/webrtc_input.py | 33 ++++++++++++++++++++++++- 5 files changed, 87 insertions(+), 1 deletion(-) diff --git a/addons/gst-web/src/app.js b/addons/gst-web/src/app.js index 0cc66d44..6edb3d40 100644 --- a/addons/gst-web/src/app.js +++ b/addons/gst-web/src/app.js @@ -578,6 +578,9 @@ window.addEventListener('focus', () => { window.addEventListener('blur', () => { // reset keyboard to avoid stuck keys. webrtc.sendDataChannelMessage("kr"); + + // Clear the key-repeat events on window blur + webrtc.input.keyRepeatQueue.clear(); }); webrtc.onclipboardcontent = (content) => { diff --git a/addons/gst-web/src/input.js b/addons/gst-web/src/input.js index e7d102bb..b2952673 100644 --- a/addons/gst-web/src/input.js +++ b/addons/gst-web/src/input.js @@ -129,6 +129,9 @@ class Input { // variable used to scale cursor speed this.cursorScaleFactor = null; + + // keys pressed list to send key repeat events to server + this.keyRepeatQueue = new Queue(); } /** @@ -610,12 +613,32 @@ class Input { this.keyboard = new Guacamole.Keyboard(window); this.keyboard.onkeydown = (keysym) => { this.send("kd," + keysym); + if (!this.keyRepeatQueue.find(keysym)) { + this.keyRepeatQueue.enqueue(keysym); + } }; this.keyboard.onkeyup = (keysym) => { this.send("ku," + keysym); + this.keyRepeatQueue.remove(keysym) }; this._windowMath(); + + this.keyRepeatRunning = true; + this._handleKeyRepeatEvents(); + } + + // A handler function to send key-repeat events for keys that are pressed and kept hold + async _handleKeyRepeatEvents(){ + while (this.keyRepeatRunning) { + var keysyms = this.keyRepeatQueue.toArray(); + + for(var keysym of keysyms){ + this.send("krpt," + keysym); + } + + await this.sleep(200); + } } detach() { @@ -628,6 +651,10 @@ class Input { delete this.keyboard; this.send("kr"); } + + // Reset the key-repeat handler + this.keyRepeatQueue.clear(); + this.keyRepeatRunning = false } /** @@ -662,6 +689,14 @@ class Input { parseInt( (() => {var offsetRatioHeight = document.body.offsetHeight * window.devicePixelRatio; return offsetRatioHeight - offsetRatioHeight % 2})() ) ]; } + + async sleep(milliseconds) { + await new Promise((resolve, reject) => { + setTimeout(() => { + resolve(); + }, milliseconds); + }); + } } /** diff --git a/addons/gst-web/src/util.js b/addons/gst-web/src/util.js index 27360092..600eff80 100644 --- a/addons/gst-web/src/util.js +++ b/addons/gst-web/src/util.js @@ -29,4 +29,20 @@ class Queue { return this.items.length===0; } + toArray() { + return [...this.items] + } + + remove(element) { + var index = this.items.indexOf(element) + this.items.splice(index, 1) + } + + find(element) { + return this.items.indexOf(element) == -1 ? false: true; + } + + clear(){ + this.items.length = 0; + } } \ No newline at end of file diff --git a/src/selkies_gstreamer/__main__.py b/src/selkies_gstreamer/__main__.py index 00daa6d1..857c03ee 100644 --- a/src/selkies_gstreamer/__main__.py +++ b/src/selkies_gstreamer/__main__.py @@ -864,6 +864,7 @@ def mon_rtc_config(stun_servers, turn_servers, rtc_config): loop.run_in_executor(None, lambda: turn_rest_mon.start()) loop.run_in_executor(None, lambda: rtc_file_mon.start()) loop.run_in_executor(None, lambda: system_mon.start()) + loop.run_in_executor(None, lambda: webrtc_input.handle_key_repeat()) while True: if using_webrtc_csv: diff --git a/src/selkies_gstreamer/webrtc_input.py b/src/selkies_gstreamer/webrtc_input.py index f4a20acd..e1d42b19 100644 --- a/src/selkies_gstreamer/webrtc_input.py +++ b/src/selkies_gstreamer/webrtc_input.py @@ -114,6 +114,9 @@ def __init__(self, uinput_mouse_socket_path="", js_socket_path="", enable_clipbo self.ping_start = None + # Stores key-repeat keys with arrival time + self.key_repeat_keys = {} + self.on_video_encoder_bit_rate = lambda bitrate: logger.warn( 'unhandled on_video_encoder_bit_rate') self.on_audio_encoder_bit_rate = lambda bitrate: logger.warn( @@ -313,7 +316,7 @@ def send_mouse(self, action, data): else: self.mouse.release(btn) - def send_x11_keypress(self, keysym, down=True): + def send_x11_keypress(self, keysym, down=True, key_repeat=False): """Sends keypress to X server The key sym is converted to a keycode using the X server library. @@ -330,11 +333,36 @@ def send_x11_keypress(self, keysym, down=True): # Although prevented in most cases, this fix may present issues in some keyboard layouts if keysym == 60 and self.keyboard._display.keysym_to_keycode(keysym) == 94: keysym = 44 + + if key_repeat: + # Set or update the timestamp of the key + self.key_repeat_keys[keysym] = time.monotonic() + return + keycode = pynput.keyboard.KeyCode(keysym) if down: self.keyboard.press(keycode) else: self.keyboard.release(keycode) + + if self.key_repeat_keys.get(keysym): + del self.key_repeat_keys[keysym] + + def handle_key_repeat(self): + """Handles key-repeat event keys by monitoring the pressed keys from the + dictionary object and releases those based on the elapsed time + """ + while True: + now = time.monotonic() + + # Iterating over a copy of the data + for key, timeout in tuple(self.key_repeat_keys.items()): + elapsed_time = now - timeout + + # Release the key if elapsed time is over 1.5s + if elapsed_time >= 1.5: + self.send_x11_keypress(key, down=False) + time.sleep(0.2) def send_x11_mouse(self, x, y, button_mask, scroll_magnitude, relative=False): """Sends mouse events to the X server. @@ -587,6 +615,9 @@ def on_message(self, msg): elif toks[0] == "kr": # Keyboard reset self.reset_keyboard() + elif toks[0] == "krpt": + # key-repreat events for a key + self.send_x11_keypress(int(toks[1]), down=False, key_repeat=True) elif toks[0] in ["m", "m2"]: # Mouse action # x,y,button_mask From 808e6d76ec2e27c4560cfccc987c79912b94db9d Mon Sep 17 00:00:00 2001 From: Seungmin Kim <8457324+ehfd@users.noreply.github.com> Date: Fri, 3 May 2024 04:15:44 +0900 Subject: [PATCH 2/2] Minor edits --- addons/gst-web/src/input.js | 2 +- src/selkies_gstreamer/webrtc_input.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/addons/gst-web/src/input.js b/addons/gst-web/src/input.js index b2952673..a2e7878d 100644 --- a/addons/gst-web/src/input.js +++ b/addons/gst-web/src/input.js @@ -634,7 +634,7 @@ class Input { var keysyms = this.keyRepeatQueue.toArray(); for(var keysym of keysyms){ - this.send("krpt," + keysym); + this.send("kt," + keysym); } await this.sleep(200); diff --git a/src/selkies_gstreamer/webrtc_input.py b/src/selkies_gstreamer/webrtc_input.py index e1d42b19..3607961b 100644 --- a/src/selkies_gstreamer/webrtc_input.py +++ b/src/selkies_gstreamer/webrtc_input.py @@ -615,8 +615,8 @@ def on_message(self, msg): elif toks[0] == "kr": # Keyboard reset self.reset_keyboard() - elif toks[0] == "krpt": - # key-repreat events for a key + elif toks[0] == "kt": + # key-repeat events for a key self.send_x11_keypress(int(toks[1]), down=False, key_repeat=True) elif toks[0] in ["m", "m2"]: # Mouse action