diff --git a/src/component/internal/control.ts b/src/component/internal/control.ts index e5ee2f12..0f2e6c66 100644 --- a/src/component/internal/control.ts +++ b/src/component/internal/control.ts @@ -26,6 +26,13 @@ export class NekoControl extends EventEmitter { super() } + get useWebrtc() { + // we want to use webrtc if we're connected and we're the host + // because webrtc is faster and it doesn't request control + // in contrast to the websocket + return this._connection.webrtc.connected && this._state.is_host + } + public lock() { Vue.set(this._state, 'locked', true) } @@ -43,35 +50,66 @@ export class NekoControl extends EventEmitter { } public move(pos: ControlPos) { - this._connection.websocket.send(EVENT.CONTROL_MOVE, pos as message.ControlPos) + if (this.useWebrtc) { + this._connection.webrtc.send('mousemove', pos) + } else { + this._connection.websocket.send(EVENT.CONTROL_MOVE, pos as message.ControlPos) + } } + // TODO: rename pos to delta, and add a new pos parameter public scroll(pos: ControlPos) { - this._connection.websocket.send(EVENT.CONTROL_SCROLL, pos as message.ControlPos) + if (this.useWebrtc) { + this._connection.webrtc.send('wheel', pos) + } else { + this._connection.websocket.send(EVENT.CONTROL_SCROLL, pos as message.ControlPos) + } } + // buttonpress ensures that only one button is pressed at a time public buttonPress(code: number, pos?: ControlPos) { this._connection.websocket.send(EVENT.CONTROL_BUTTONPRESS, { code, ...pos } as message.ControlButton) } public buttonDown(code: number, pos?: ControlPos) { - this._connection.websocket.send(EVENT.CONTROL_BUTTONDOWN, { code, ...pos } as message.ControlButton) + if (this.useWebrtc) { + if (pos) this._connection.webrtc.send('mousemove', pos) + this._connection.webrtc.send('mousedown', { key: code }) + } else { + this._connection.websocket.send(EVENT.CONTROL_BUTTONDOWN, { code, ...pos } as message.ControlButton) + } } public buttonUp(code: number, pos?: ControlPos) { - this._connection.websocket.send(EVENT.CONTROL_BUTTONUP, { code, ...pos } as message.ControlButton) + if (this.useWebrtc) { + if (pos) this._connection.webrtc.send('mousemove', pos) + this._connection.webrtc.send('mouseup', { key: code }) + } else { + this._connection.websocket.send(EVENT.CONTROL_BUTTONUP, { code, ...pos } as message.ControlButton) + } } + // keypress ensures that only one key is pressed at a time public keyPress(keysym: number, pos?: ControlPos) { this._connection.websocket.send(EVENT.CONTROL_KEYPRESS, { keysym, ...pos } as message.ControlKey) } public keyDown(keysym: number, pos?: ControlPos) { - this._connection.websocket.send(EVENT.CONTROL_KEYDOWN, { keysym, ...pos } as message.ControlKey) + if (this.useWebrtc) { + if (pos) this._connection.webrtc.send('mousemove', pos) + this._connection.webrtc.send('keydown', { key: keysym }) + } else { + this._connection.websocket.send(EVENT.CONTROL_KEYDOWN, { keysym, ...pos } as message.ControlKey) + } } public keyUp(keysym: number, pos?: ControlPos) { - this._connection.websocket.send(EVENT.CONTROL_KEYUP, { keysym, ...pos } as message.ControlKey) + if (this.useWebrtc) { + if (pos) this._connection.webrtc.send('mousemove', pos) + this._connection.webrtc.send('keyup', { key: keysym }) + } else { + this._connection.websocket.send(EVENT.CONTROL_KEYUP, { keysym, ...pos } as message.ControlKey) + } } public cut() { diff --git a/src/component/internal/messages.ts b/src/component/internal/messages.ts index 42147520..7080d1aa 100644 --- a/src/component/internal/messages.ts +++ b/src/component/internal/messages.ts @@ -262,6 +262,9 @@ export class NekoMessages extends EventEmitter { Vue.set(this._state.control, 'host_id', null) } + // save if user is host + Vue.set(this._state.control, 'is_host', has_host && this._state.control.host_id === this._state.session_id) + this.emit('room.control.host', has_host, host_id) } diff --git a/src/component/main.vue b/src/component/main.vue index 057cb224..dee9ce32 100644 --- a/src/component/main.vue +++ b/src/component/main.vue @@ -24,7 +24,7 @@ ref="overlay" v-show="!private_mode_enabled && state.connection.status != 'disconnected'" :style="{ pointerEvents: state.control.locked ? 'none' : 'auto' }" - :wsControl="control" + :control="control" :sessions="state.sessions" :hostId="state.control.host_id" :webrtc="connection.webrtc" @@ -201,6 +201,7 @@ variant: '', }, host_id: null, + is_host: false, locked: false, }, screen: { @@ -763,6 +764,7 @@ // websocket Vue.set(this.state.control, 'clipboard', null) Vue.set(this.state.control, 'host_id', null) + Vue.set(this.state.control, 'is_host', false) Vue.set(this.state.screen, 'size', { width: 1280, height: 720, rate: 30 }) Vue.set(this.state.screen, 'configurations', []) Vue.set(this.state.screen, 'sync', false) diff --git a/src/component/overlay.vue b/src/component/overlay.vue index 050ae2dc..e5c56416 100644 --- a/src/component/overlay.vue +++ b/src/component/overlay.vue @@ -6,8 +6,8 @@ class="neko-overlay" :style="{ cursor }" v-model="textInput" - @click.stop.prevent="wsControl.emit('overlay.click', $event)" - @contextmenu.stop.prevent="wsControl.emit('overlay.contextmenu', $event)" + @click.stop.prevent="control.emit('overlay.click', $event)" + @contextmenu.stop.prevent="control.emit('overlay.contextmenu', $event)" @wheel.stop.prevent="onWheel" @mousemove.stop.prevent="onMouseMove" @mousedown.stop.prevent="onMouseDown" @@ -47,6 +47,7 @@ import { Vue, Component, Ref, Prop, Watch } from 'vue-property-decorator' import { KeyboardInterface, NewKeyboard } from './utils/keyboard' + import GestureHandlerInit, { GestureHandler } from './utils/gesturehandler' import { KeyTable, keySymsRemap } from './utils/keyboard-remapping' import { getFilesFromDataTansfer } from './utils/file-upload' import { NekoControl } from './internal/control' @@ -55,8 +56,15 @@ import { CursorPosition, CursorImage } from './types/webrtc' import { CursorDrawFunction, Dimension, KeyboardModifiers } from './types/cursors' - const WHEEL_STEP = 53 // Delta threshold for a mouse wheel step - const WHEEL_LINE_HEIGHT = 19 + // Wheel thresholds + const WHEEL_STEP = 53 // Pixels needed for one step + const WHEEL_LINE_HEIGHT = 19 // Assumed pixels for one line step + + // Gesture thresholds + const GESTURE_ZOOMSENS = 75 + const GESTURE_SCRLSENS = 50 + const DOUBLE_TAP_TIMEOUT = 1000 + const DOUBLE_TAP_THRESHOLD = 50 const MOUSE_MOVE_THROTTLE = 1000 / 60 // in ms, 60fps const INACTIVE_CURSOR_INTERVAL = 1000 / 4 // in ms, 4fps @@ -72,12 +80,13 @@ private canvasScale = window.devicePixelRatio private keyboard!: KeyboardInterface + private gestureHandler!: GestureHandler private textInput = '' private focused = false @Prop() - private readonly wsControl!: NekoControl + private readonly control!: NekoControl @Prop() private readonly sessions!: Record @@ -165,12 +174,7 @@ const isCtrlKey = key == KeyTable.XK_Control_L || key == KeyTable.XK_Control_R if (isCtrlKey) ctrlKey = key - if (this.webrtc.connected) { - this.webrtc.send('keydown', { key }) - } else { - this.wsControl.keyDown(key) - } - + this.control.keyDown(key) return isCtrlKey } this.keyboard.onkeyup = (key: number) => { @@ -184,17 +188,17 @@ const isCtrlKey = key == KeyTable.XK_Control_L || key == KeyTable.XK_Control_R if (isCtrlKey) ctrlKey = 0 - if (this.webrtc.connected) { - this.webrtc.send('keyup', { key }) - } else { - this.wsControl.keyUp(key) - } + this.control.keyUp(key) } this.keyboard.listenTo(this._textarea) - this._textarea.addEventListener('touchstart', this.onTouchHandler, { passive: false }) - this._textarea.addEventListener('touchmove', this.onTouchHandler, { passive: false }) - this._textarea.addEventListener('touchend', this.onTouchHandler, { passive: false }) + // Initialize GestureHandler + this.gestureHandler = new GestureHandlerInit() + this.gestureHandler.attach(this._textarea) + + this._textarea.addEventListener('gesturestart', this.onGestureHandler) + this._textarea.addEventListener('gesturemove', this.onGestureHandler) + this._textarea.addEventListener('gestureend', this.onGestureHandler) this.webrtc.addListener('cursor-position', this.onCursorPosition) this.webrtc.addListener('cursor-image', this.onCursorImage) @@ -209,9 +213,13 @@ this.keyboard.removeListener() } - this._textarea.removeEventListener('touchstart', this.onTouchHandler) - this._textarea.removeEventListener('touchmove', this.onTouchHandler) - this._textarea.removeEventListener('touchend', this.onTouchHandler) + if (this.gestureHandler) { + this.gestureHandler.detach() + } + + this._textarea.removeEventListener('gesturestart', this.onGestureHandler) + this._textarea.removeEventListener('gesturemove', this.onGestureHandler) + this._textarea.removeEventListener('gestureend', this.onGestureHandler) this.webrtc.removeListener('cursor-position', this.onCursorPosition) this.webrtc.removeListener('cursor-image', this.onCursorImage) @@ -227,34 +235,161 @@ } } - onTouchHandler(e: TouchEvent) { - let type = '' - switch (e.type) { - case 'touchstart': - type = 'mousedown' - break - case 'touchmove': - type = 'mousemove' - break - case 'touchend': - type = 'mouseup' - break - default: - // unknown event - return + // Gesture state + private _gestureLastTapTime: any | null = null + private _gestureFirstDoubleTapEv: any | null = null + private _gestureLastMagnitudeX = 0 + private _gestureLastMagnitudeY = 0 + + _handleTapEvent(ev: any, code: number) { + let pos = this.getMousePos(ev.detail.clientX, ev.detail.clientY) + + // If the user quickly taps multiple times we assume they meant to + // hit the same spot, so slightly adjust coordinates + + if ( + this._gestureLastTapTime !== null && + Date.now() - this._gestureLastTapTime < DOUBLE_TAP_TIMEOUT && + this._gestureFirstDoubleTapEv.detail.type === ev.detail.type + ) { + let dx = this._gestureFirstDoubleTapEv.detail.clientX - ev.detail.clientX + let dy = this._gestureFirstDoubleTapEv.detail.clientY - ev.detail.clientY + let distance = Math.hypot(dx, dy) + + if (distance < DOUBLE_TAP_THRESHOLD) { + pos = this.getMousePos( + this._gestureFirstDoubleTapEv.detail.clientX, + this._gestureFirstDoubleTapEv.detail.clientY, + ) + } else { + this._gestureFirstDoubleTapEv = ev + } + } else { + this._gestureFirstDoubleTapEv = ev + } + this._gestureLastTapTime = Date.now() + + this.control.buttonDown(code, pos) + this.control.buttonUp(code, pos) + } + + // https://github.com/novnc/noVNC/blob/ca6527c1bf7131adccfdcc5028964a1e67f9018c/core/rfb.js#L1227-L1345 + onGestureHandler(ev: any) { + // we cannot use implicitControlRequest because we don't have mouse event + if (!this.isControling) { + // if implicitControl is enabled, request control + if (this.implicitControl) { + this.control.request() + } + // otherwise, ignore event + return } - const touch = e.changedTouches[0] - touch.target.dispatchEvent( - new MouseEvent(type, { - button: 0, // currently only left button is supported - clientX: touch.clientX, - clientY: touch.clientY, - }), - ) + const pos = this.getMousePos(ev.detail.clientX, ev.detail.clientY) - e.preventDefault() - e.stopPropagation() + let magnitude + switch (ev.type) { + case 'gesturestart': + switch (ev.detail.type) { + case 'onetap': + this._handleTapEvent(ev, 1) + break + case 'twotap': + this._handleTapEvent(ev, 3) + break + case 'threetap': + this._handleTapEvent(ev, 2) + break + case 'drag': + this.control.buttonDown(1, pos) + break + case 'longpress': + this.control.buttonDown(3, pos) + break + + case 'twodrag': + this._gestureLastMagnitudeX = ev.detail.magnitudeX + this._gestureLastMagnitudeY = ev.detail.magnitudeY + this.control.move(pos) + break + case 'pinch': + this._gestureLastMagnitudeX = Math.hypot(ev.detail.magnitudeX, ev.detail.magnitudeY) + this.control.move(pos) + break + } + break + + case 'gesturemove': + switch (ev.detail.type) { + case 'onetap': + case 'twotap': + case 'threetap': + break + case 'drag': + case 'longpress': + this.control.move(pos) + break + case 'twodrag': + // Always scroll in the same position. + // We don't know if the mouse was moved so we need to move it + // every update. + this.control.move(pos) + while (ev.detail.magnitudeY - this._gestureLastMagnitudeY > GESTURE_SCRLSENS) { + this.control.scroll({ x: 0, y: 1 }) + this._gestureLastMagnitudeY += GESTURE_SCRLSENS + } + while (ev.detail.magnitudeY - this._gestureLastMagnitudeY < -GESTURE_SCRLSENS) { + this.control.scroll({ x: 0, y: -1 }) + this._gestureLastMagnitudeY -= GESTURE_SCRLSENS + } + while (ev.detail.magnitudeX - this._gestureLastMagnitudeX > GESTURE_SCRLSENS) { + this.control.scroll({ x: 1, y: 0 }) + this._gestureLastMagnitudeX += GESTURE_SCRLSENS + } + while (ev.detail.magnitudeX - this._gestureLastMagnitudeX < -GESTURE_SCRLSENS) { + this.control.scroll({ x: -1, y: 0 }) + this._gestureLastMagnitudeX -= GESTURE_SCRLSENS + } + break + case 'pinch': + // Always scroll in the same position. + // We don't know if the mouse was moved so we need to move it + // every update. + this.control.move(pos) + magnitude = Math.hypot(ev.detail.magnitudeX, ev.detail.magnitudeY) + if (Math.abs(magnitude - this._gestureLastMagnitudeX) > GESTURE_ZOOMSENS) { + this.control.keyDown(KeyTable.XK_Control_L) + while (magnitude - this._gestureLastMagnitudeX > GESTURE_ZOOMSENS) { + this.control.scroll({ x: 0, y: 1 }) + this._gestureLastMagnitudeX += GESTURE_ZOOMSENS + } + while (magnitude - this._gestureLastMagnitudeX < -GESTURE_ZOOMSENS) { + this.control.scroll({ x: 0, y: -1 }) + this._gestureLastMagnitudeX -= GESTURE_ZOOMSENS + } + this.control.keyUp(KeyTable.XK_Control_L) + } + break + } + break + + case 'gestureend': + switch (ev.detail.type) { + case 'onetap': + case 'twotap': + case 'threetap': + case 'pinch': + case 'twodrag': + break + case 'drag': + this.control.buttonUp(1, pos) + break + case 'longpress': + this.control.buttonUp(3, pos) + break + } + break + } } getMousePos(clientX: number, clientY: number) { @@ -268,7 +403,11 @@ sendMousePos(e: MouseEvent) { const pos = this.getMousePos(e.clientX, e.clientY) - this.webrtc.send('mousemove', pos) + // not using NekoControl here because we want to avoid + // sending mousemove events over websocket + if (this.webrtc.connected) { + this.webrtc.send('mousemove', pos) + } // otherwise, no events are sent this.cursorPosition = pos } @@ -307,7 +446,7 @@ @Watch('textInput') onTextInputChange() { if (this.textInput == '') return - this.wsControl.paste(this.textInput) + this.control.paste(this.textInput) this.textInput = '' } @@ -365,12 +504,8 @@ // skip if not scrolled if (x == 0 && y == 0) return - if (this.webrtc.connected) { - this.sendMousePos(e) - this.webrtc.send('wheel', { x, y }) - } else { - this.wsControl.scroll({ x, y }) - } + // TODO: add position for precision scrolling + this.control.scroll({ x, y }) } lastMouseMove = 0 @@ -400,13 +535,8 @@ } const key = e.button + 1 - if (this.webrtc.connected) { - this.sendMousePos(e) - this.webrtc.send('mousedown', { key }) - } else { - const pos = this.getMousePos(e.clientX, e.clientY) - this.wsControl.buttonDown(key, pos) - } + const pos = this.getMousePos(e.clientX, e.clientY) + this.control.buttonDown(key, pos) } onMouseUp(e: MouseEvent) { @@ -420,13 +550,8 @@ } const key = e.button + 1 - if (this.webrtc.connected) { - this.sendMousePos(e) - this.webrtc.send('mouseup', { key }) - } else { - const pos = this.getMousePos(e.clientX, e.clientY) - this.wsControl.buttonUp(key, pos) - } + const pos = this.getMousePos(e.clientX, e.clientY) + this.control.buttonUp(key, pos) } onMouseEnter(e: MouseEvent) { @@ -474,7 +599,8 @@ const files = await getFilesFromDataTansfer(dt) if (files.length === 0) return - this.$emit('uploadDrop', { ...this.getMousePos(e.clientX, e.clientY), files }) + const pos = this.getMousePos(e.clientX, e.clientY) + this.$emit('uploadDrop', { ...pos, files }) } } @@ -510,9 +636,11 @@ } sendInactiveMousePos() { - if (this.inactiveCursorPosition) { + if (this.inactiveCursorPosition && this.webrtc.connected) { + // not using NekoControl here, because inactive cursors are + // treated differently than moving the mouse while controling this.webrtc.send('mousemove', this.inactiveCursorPosition) - } + } // if webrtc is not connected, we don't need to send anything } // @@ -715,7 +843,7 @@ if (this.implicitControl && e.type === 'mousedown') { this.reqMouseDown = e this.reqMouseUp = null - this.wsControl.request() + this.control.request() } if (this.implicitControl && e.type === 'mouseup') { @@ -726,7 +854,7 @@ // unused implicitControlRelease() { if (this.implicitControl) { - this.wsControl.release() + this.control.release() } } diff --git a/src/component/types/state.ts b/src/component/types/state.ts index 70ce4728..b9ed4b64 100644 --- a/src/component/types/state.ts +++ b/src/component/types/state.ts @@ -71,6 +71,7 @@ export interface Control { clipboard: Clipboard | null keyboard: Keyboard host_id: string | null + is_host: boolean locked: boolean } diff --git a/src/component/utils/gesturehandler.js b/src/component/utils/gesturehandler.js new file mode 100644 index 00000000..6fa72d2a --- /dev/null +++ b/src/component/utils/gesturehandler.js @@ -0,0 +1,567 @@ +/* + * noVNC: HTML5 VNC client + * Copyright (C) 2020 The noVNC Authors + * Licensed under MPL 2.0 (see LICENSE.txt) + * + * See README.md for usage and integration instructions. + * + */ + +const GH_NOGESTURE = 0; +const GH_ONETAP = 1; +const GH_TWOTAP = 2; +const GH_THREETAP = 4; +const GH_DRAG = 8; +const GH_LONGPRESS = 16; +const GH_TWODRAG = 32; +const GH_PINCH = 64; + +const GH_INITSTATE = 127; + +const GH_MOVE_THRESHOLD = 50; +const GH_ANGLE_THRESHOLD = 90; // Degrees + +// Timeout when waiting for gestures (ms) +const GH_MULTITOUCH_TIMEOUT = 250; + +// Maximum time between press and release for a tap (ms) +const GH_TAP_TIMEOUT = 1000; + +// Timeout when waiting for longpress (ms) +const GH_LONGPRESS_TIMEOUT = 1000; + +// Timeout when waiting to decide between PINCH and TWODRAG (ms) +const GH_TWOTOUCH_TIMEOUT = 50; + +export default class GestureHandler { + constructor() { + this._target = null; + + this._state = GH_INITSTATE; + + this._tracked = []; + this._ignored = []; + + this._waitingRelease = false; + this._releaseStart = 0.0; + + this._longpressTimeoutId = null; + this._twoTouchTimeoutId = null; + + this._boundEventHandler = this._eventHandler.bind(this); + } + + attach(target) { + this.detach(); + + this._target = target; + this._target.addEventListener('touchstart', + this._boundEventHandler); + this._target.addEventListener('touchmove', + this._boundEventHandler); + this._target.addEventListener('touchend', + this._boundEventHandler); + this._target.addEventListener('touchcancel', + this._boundEventHandler); + } + + detach() { + if (!this._target) { + return; + } + + this._stopLongpressTimeout(); + this._stopTwoTouchTimeout(); + + this._target.removeEventListener('touchstart', + this._boundEventHandler); + this._target.removeEventListener('touchmove', + this._boundEventHandler); + this._target.removeEventListener('touchend', + this._boundEventHandler); + this._target.removeEventListener('touchcancel', + this._boundEventHandler); + this._target = null; + } + + _eventHandler(e) { + let fn; + + e.stopPropagation(); + e.preventDefault(); + + switch (e.type) { + case 'touchstart': + fn = this._touchStart; + break; + case 'touchmove': + fn = this._touchMove; + break; + case 'touchend': + case 'touchcancel': + fn = this._touchEnd; + break; + } + + for (let i = 0; i < e.changedTouches.length; i++) { + let touch = e.changedTouches[i]; + fn.call(this, touch.identifier, touch.clientX, touch.clientY); + } + } + + _touchStart(id, x, y) { + // Ignore any new touches if there is already an active gesture, + // or we're in a cleanup state + if (this._hasDetectedGesture() || (this._state === GH_NOGESTURE)) { + this._ignored.push(id); + return; + } + + // Did it take too long between touches that we should no longer + // consider this a single gesture? + if ((this._tracked.length > 0) && + ((Date.now() - this._tracked[0].started) > GH_MULTITOUCH_TIMEOUT)) { + this._state = GH_NOGESTURE; + this._ignored.push(id); + return; + } + + // If we're waiting for fingers to release then we should no longer + // recognize new touches + if (this._waitingRelease) { + this._state = GH_NOGESTURE; + this._ignored.push(id); + return; + } + + this._tracked.push({ + id: id, + started: Date.now(), + active: true, + firstX: x, + firstY: y, + lastX: x, + lastY: y, + angle: 0 + }); + + switch (this._tracked.length) { + case 1: + this._startLongpressTimeout(); + break; + + case 2: + this._state &= ~(GH_ONETAP | GH_DRAG | GH_LONGPRESS); + this._stopLongpressTimeout(); + break; + + case 3: + this._state &= ~(GH_TWOTAP | GH_TWODRAG | GH_PINCH); + break; + + default: + this._state = GH_NOGESTURE; + } + } + + _touchMove(id, x, y) { + let touch = this._tracked.find(t => t.id === id); + + // If this is an update for a touch we're not tracking, ignore it + if (touch === undefined) { + return; + } + + // Update the touches last position with the event coordinates + touch.lastX = x; + touch.lastY = y; + + let deltaX = x - touch.firstX; + let deltaY = y - touch.firstY; + + // Update angle when the touch has moved + if ((touch.firstX !== touch.lastX) || + (touch.firstY !== touch.lastY)) { + touch.angle = Math.atan2(deltaY, deltaX) * 180 / Math.PI; + } + + if (!this._hasDetectedGesture()) { + // Ignore moves smaller than the minimum threshold + if (Math.hypot(deltaX, deltaY) < GH_MOVE_THRESHOLD) { + return; + } + + // Can't be a tap or long press as we've seen movement + this._state &= ~(GH_ONETAP | GH_TWOTAP | GH_THREETAP | GH_LONGPRESS); + this._stopLongpressTimeout(); + + if (this._tracked.length !== 1) { + this._state &= ~(GH_DRAG); + } + if (this._tracked.length !== 2) { + this._state &= ~(GH_TWODRAG | GH_PINCH); + } + + // We need to figure out which of our different two touch gestures + // this might be + if (this._tracked.length === 2) { + + // The other touch is the one where the id doesn't match + let prevTouch = this._tracked.find(t => t.id !== id); + + // How far the previous touch point has moved since start + let prevDeltaMove = Math.hypot(prevTouch.firstX - prevTouch.lastX, + prevTouch.firstY - prevTouch.lastY); + + // We know that the current touch moved far enough, + // but unless both touches moved further than their + // threshold we don't want to disqualify any gestures + if (prevDeltaMove > GH_MOVE_THRESHOLD) { + + // The angle difference between the direction of the touch points + let deltaAngle = Math.abs(touch.angle - prevTouch.angle); + deltaAngle = Math.abs(((deltaAngle + 180) % 360) - 180); + + // PINCH or TWODRAG can be eliminated depending on the angle + if (deltaAngle > GH_ANGLE_THRESHOLD) { + this._state &= ~GH_TWODRAG; + } else { + this._state &= ~GH_PINCH; + } + + if (this._isTwoTouchTimeoutRunning()) { + this._stopTwoTouchTimeout(); + } + } else if (!this._isTwoTouchTimeoutRunning()) { + // We can't determine the gesture right now, let's + // wait and see if more events are on their way + this._startTwoTouchTimeout(); + } + } + + if (!this._hasDetectedGesture()) { + return; + } + + this._pushEvent('gesturestart'); + } + + this._pushEvent('gesturemove'); + } + + _touchEnd(id, x, y) { + // Check if this is an ignored touch + if (this._ignored.indexOf(id) !== -1) { + // Remove this touch from ignored + this._ignored.splice(this._ignored.indexOf(id), 1); + + // And reset the state if there are no more touches + if ((this._ignored.length === 0) && + (this._tracked.length === 0)) { + this._state = GH_INITSTATE; + this._waitingRelease = false; + } + return; + } + + // We got a touchend before the timer triggered, + // this cannot result in a gesture anymore. + if (!this._hasDetectedGesture() && + this._isTwoTouchTimeoutRunning()) { + this._stopTwoTouchTimeout(); + this._state = GH_NOGESTURE; + } + + // Some gestures don't trigger until a touch is released + if (!this._hasDetectedGesture()) { + // Can't be a gesture that relies on movement + this._state &= ~(GH_DRAG | GH_TWODRAG | GH_PINCH); + // Or something that relies on more time + this._state &= ~GH_LONGPRESS; + this._stopLongpressTimeout(); + + if (!this._waitingRelease) { + this._releaseStart = Date.now(); + this._waitingRelease = true; + + // Can't be a tap that requires more touches than we current have + switch (this._tracked.length) { + case 1: + this._state &= ~(GH_TWOTAP | GH_THREETAP); + break; + + case 2: + this._state &= ~(GH_ONETAP | GH_THREETAP); + break; + } + } + } + + // Waiting for all touches to release? (i.e. some tap) + if (this._waitingRelease) { + // Were all touches released at roughly the same time? + if ((Date.now() - this._releaseStart) > GH_MULTITOUCH_TIMEOUT) { + this._state = GH_NOGESTURE; + } + + // Did too long time pass between press and release? + if (this._tracked.some(t => (Date.now() - t.started) > GH_TAP_TIMEOUT)) { + this._state = GH_NOGESTURE; + } + + let touch = this._tracked.find(t => t.id === id); + touch.active = false; + + // Are we still waiting for more releases? + if (this._hasDetectedGesture()) { + this._pushEvent('gesturestart'); + } else { + // Have we reached a dead end? + if (this._state !== GH_NOGESTURE) { + return; + } + } + } + + if (this._hasDetectedGesture()) { + this._pushEvent('gestureend'); + } + + // Ignore any remaining touches until they are ended + for (let i = 0; i < this._tracked.length; i++) { + if (this._tracked[i].active) { + this._ignored.push(this._tracked[i].id); + } + } + this._tracked = []; + + this._state = GH_NOGESTURE; + + // Remove this touch from ignored if it's in there + if (this._ignored.indexOf(id) !== -1) { + this._ignored.splice(this._ignored.indexOf(id), 1); + } + + // We reset the state if ignored is empty + if ((this._ignored.length === 0)) { + this._state = GH_INITSTATE; + this._waitingRelease = false; + } + } + + _hasDetectedGesture() { + if (this._state === GH_NOGESTURE) { + return false; + } + // Check to see if the bitmask value is a power of 2 + // (i.e. only one bit set). If it is, we have a state. + if (this._state & (this._state - 1)) { + return false; + } + + // For taps we also need to have all touches released + // before we've fully detected the gesture + if (this._state & (GH_ONETAP | GH_TWOTAP | GH_THREETAP)) { + if (this._tracked.some(t => t.active)) { + return false; + } + } + + return true; + } + + _startLongpressTimeout() { + this._stopLongpressTimeout(); + this._longpressTimeoutId = setTimeout(() => this._longpressTimeout(), + GH_LONGPRESS_TIMEOUT); + } + + _stopLongpressTimeout() { + clearTimeout(this._longpressTimeoutId); + this._longpressTimeoutId = null; + } + + _longpressTimeout() { + if (this._hasDetectedGesture()) { + throw new Error("A longpress gesture failed, conflict with a different gesture"); + } + + this._state = GH_LONGPRESS; + this._pushEvent('gesturestart'); + } + + _startTwoTouchTimeout() { + this._stopTwoTouchTimeout(); + this._twoTouchTimeoutId = setTimeout(() => this._twoTouchTimeout(), + GH_TWOTOUCH_TIMEOUT); + } + + _stopTwoTouchTimeout() { + clearTimeout(this._twoTouchTimeoutId); + this._twoTouchTimeoutId = null; + } + + _isTwoTouchTimeoutRunning() { + return this._twoTouchTimeoutId !== null; + } + + _twoTouchTimeout() { + if (this._tracked.length === 0) { + throw new Error("A pinch or two drag gesture failed, no tracked touches"); + } + + // How far each touch point has moved since start + let avgM = this._getAverageMovement(); + let avgMoveH = Math.abs(avgM.x); + let avgMoveV = Math.abs(avgM.y); + + // The difference in the distance between where + // the touch points started and where they are now + let avgD = this._getAverageDistance(); + let deltaTouchDistance = Math.abs(Math.hypot(avgD.first.x, avgD.first.y) - + Math.hypot(avgD.last.x, avgD.last.y)); + + if ((avgMoveV < deltaTouchDistance) && + (avgMoveH < deltaTouchDistance)) { + this._state = GH_PINCH; + } else { + this._state = GH_TWODRAG; + } + + this._pushEvent('gesturestart'); + this._pushEvent('gesturemove'); + } + + _pushEvent(type) { + let detail = { type: this._stateToGesture(this._state) }; + + // For most gesture events the current (average) position is the + // most useful + let avg = this._getPosition(); + let pos = avg.last; + + // However we have a slight distance to detect gestures, so for the + // first gesture event we want to use the first positions we saw + if (type === 'gesturestart') { + pos = avg.first; + } + + // For these gestures, we always want the event coordinates + // to be where the gesture began, not the current touch location. + switch (this._state) { + case GH_TWODRAG: + case GH_PINCH: + pos = avg.first; + break; + } + + detail['clientX'] = pos.x; + detail['clientY'] = pos.y; + + // FIXME: other coordinates? + + // Some gestures also have a magnitude + if (this._state === GH_PINCH) { + let distance = this._getAverageDistance(); + if (type === 'gesturestart') { + detail['magnitudeX'] = distance.first.x; + detail['magnitudeY'] = distance.first.y; + } else { + detail['magnitudeX'] = distance.last.x; + detail['magnitudeY'] = distance.last.y; + } + } else if (this._state === GH_TWODRAG) { + if (type === 'gesturestart') { + detail['magnitudeX'] = 0.0; + detail['magnitudeY'] = 0.0; + } else { + let movement = this._getAverageMovement(); + detail['magnitudeX'] = movement.x; + detail['magnitudeY'] = movement.y; + } + } + + let gev = new CustomEvent(type, { detail: detail }); + this._target.dispatchEvent(gev); + } + + _stateToGesture(state) { + switch (state) { + case GH_ONETAP: + return 'onetap'; + case GH_TWOTAP: + return 'twotap'; + case GH_THREETAP: + return 'threetap'; + case GH_DRAG: + return 'drag'; + case GH_LONGPRESS: + return 'longpress'; + case GH_TWODRAG: + return 'twodrag'; + case GH_PINCH: + return 'pinch'; + } + + throw new Error("Unknown gesture state: " + state); + } + + _getPosition() { + if (this._tracked.length === 0) { + throw new Error("Failed to get gesture position, no tracked touches"); + } + + let size = this._tracked.length; + let fx = 0, fy = 0, lx = 0, ly = 0; + + for (let i = 0; i < this._tracked.length; i++) { + fx += this._tracked[i].firstX; + fy += this._tracked[i].firstY; + lx += this._tracked[i].lastX; + ly += this._tracked[i].lastY; + } + + return { first: { x: fx / size, + y: fy / size }, + last: { x: lx / size, + y: ly / size } }; + } + + _getAverageMovement() { + if (this._tracked.length === 0) { + throw new Error("Failed to get gesture movement, no tracked touches"); + } + + let totalH, totalV; + totalH = totalV = 0; + let size = this._tracked.length; + + for (let i = 0; i < this._tracked.length; i++) { + totalH += this._tracked[i].lastX - this._tracked[i].firstX; + totalV += this._tracked[i].lastY - this._tracked[i].firstY; + } + + return { x: totalH / size, + y: totalV / size }; + } + + _getAverageDistance() { + if (this._tracked.length === 0) { + throw new Error("Failed to get gesture distance, no tracked touches"); + } + + // Distance between the first and last tracked touches + + let first = this._tracked[0]; + let last = this._tracked[this._tracked.length - 1]; + + let fdx = Math.abs(last.firstX - first.firstX); + let fdy = Math.abs(last.firstY - first.firstY); + + let ldx = Math.abs(last.lastX - first.lastX); + let ldy = Math.abs(last.lastY - first.lastY); + + return { first: { x: fdx, y: fdy }, + last: { x: ldx, y: ldy } }; + } +} diff --git a/src/component/utils/gesturehandler.ts b/src/component/utils/gesturehandler.ts new file mode 100644 index 00000000..93053be4 --- /dev/null +++ b/src/component/utils/gesturehandler.ts @@ -0,0 +1,14 @@ +// https://github.com/novnc/noVNC/blob/ca6527c1bf7131adccfdcc5028964a1e67f9018c/core/input/gesturehandler.js#L246 +import gh from './gesturehandler.js' + +const g = gh as GestureHandlerConstructor +export default g + +interface GestureHandlerConstructor { + new (): GestureHandler +} + +export interface GestureHandler { + attach(element: Element): void + detach(): void +} diff --git a/src/component/utils/keyboards/guacamole.ts b/src/component/utils/keyboards/guacamole.ts index ed89b89a..816b1bdf 100644 --- a/src/component/utils/keyboards/guacamole.ts +++ b/src/component/utils/keyboards/guacamole.ts @@ -1,3 +1,4 @@ +// https://github.com/apache/guacamole-client/blob/1ca1161a68030565a37319ec6275556dfcd1a1af/guacamole-common-js/src/main/webapp/modules/Keyboard.js import GuacamoleKeyboard from './guacamole.js' export interface GuacamoleKeyboardInterface { diff --git a/src/component/utils/keyboards/novnc.ts b/src/component/utils/keyboards/novnc.ts index f5756c20..a7966b6e 100644 --- a/src/component/utils/keyboards/novnc.ts +++ b/src/component/utils/keyboards/novnc.ts @@ -1,3 +1,4 @@ +// https://github.com/novnc/noVNC/blob/ca6527c1bf7131adccfdcc5028964a1e67f9018c/core/input/keyboard.js import Keyboard from './novnc.js' export interface NoVncKeyboardInterface {