Skip to content

Commit f61ec2d

Browse files
shamanecshamanec
authored
Improve landscape handling (shamanec#125)
1 parent 903a119 commit f61ec2d

File tree

5 files changed

+169
-103
lines changed

5 files changed

+169
-103
lines changed

‎hub/gads-ui/src/components/DeviceControl/DeviceControl.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ export default function DeviceControl() {
5151
wsType = "wss"
5252
}
5353
let socketUrl = `${wsType}://${window.location.host}/devices/control/${udid}/in-use`
54-
// let socketUrl = `${wsType}://192.168.68.109:10000/devices/control/${udid}/in-use`
54+
// let socketUrl = `${wsType}://192.168.1.41:10000/devices/control/${udid}/in-use`
5555
in_use_socket = new WebSocket(socketUrl)
5656
in_use_socket.onopen = () => {
5757
console.log('In Use WebSocket connection opened');

‎hub/gads-ui/src/components/DeviceControl/StreamCanvas.js

+113-65
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,10 @@ export default function StreamCanvas({ deviceData }) {
3939

4040
let streamUrl = ""
4141
if (deviceData.os === 'ios') {
42-
// streamUrl = `http://192.168.1.6:10000/device/${deviceData.udid}/ios-stream-mjpeg`
42+
// streamUrl = `http://192.168.1.41:10000/device/${deviceData.udid}/ios-stream-mjpeg`
4343
streamUrl = `/device/${deviceData.udid}/ios-stream-mjpeg`
4444
} else {
45-
// streamUrl = `http://192.168.1.6:10000/device/${deviceData.udid}/android-stream-mjpeg`
45+
// streamUrl = `http://192.168.1.41:10000/device/${deviceData.udid}/android-stream-mjpeg`
4646
streamUrl = `/device/${deviceData.udid}/android-stream-mjpeg`
4747
}
4848

@@ -224,47 +224,28 @@ export default function StreamCanvas({ deviceData }) {
224224

225225
function Canvas({ authToken, logout, streamData, setDialog }) {
226226
var tapStartAt = 0
227-
var coord1;
228-
var coord2;
227+
var coord1
228+
var coord2
229229

230230
function getCursorCoordinates(event) {
231231
const rect = event.currentTarget.getBoundingClientRect()
232-
// If its portrait use the usual calculation for tap coordinates
233-
if (streamData.isPortrait) {
234-
const x = event.clientX - rect.left
235-
const y = event.clientY - rect.top
236-
return [x, y]
237-
}
238-
// If its landscape and the provider uses custom wda for tapping/swiping
239-
if (streamData.device_os === 'ios' && streamData.uses_custom_wda) {
240-
// Have to subtract the tap coordinate from the canvas height
241-
// Because the custom wda tap uses coordinates where in landscape
242-
// x starts from the bottom(being essentially reversed y)
243-
// and y starts from the left(being essentially x)
244-
const x = streamData.canvasHeight - (event.clientY - rect.top)
245-
const y = event.clientX - rect.left
246-
return [x, y]
247-
}
248-
// If its landscape and provider does not use custom wda for tapping/swiping
249-
// just reverse x and y
250-
const x = event.clientY - rect.top
251-
const y = event.clientX - rect.left
252-
console.log("Returning " + x + " " + y)
253-
return [y, x];
232+
const x = event.clientX - rect.left
233+
const y = event.clientY - rect.top
234+
return [x, y];
254235
}
255236

256237
function handleMouseDown(event) {
257238
tapStartAt = (new Date()).getTime()
258239
coord1 = getCursorCoordinates(event)
259-
console.log('Tapped on ' + coord1)
260240
}
261241

262242
function handleMouseUp(event) {
263243
coord2 = getCursorCoordinates(event)
264-
console.log('Released on' + coord2)
265244
// get the time of finishing the click on the canvas
266245
var tapEndAt = (new Date()).getTime()
267246

247+
console.log('Tap on ' + coord1[0] + 'x' + coord1[1] + ', released on ' + coord2[0] + 'x' + coord2[1])
248+
268249
var mouseEventsTimeDiff = tapEndAt - tapStartAt
269250

270251
// if the difference of time between click down and click up is more than 500ms assume it is a swipe, not a tap
@@ -275,7 +256,7 @@ function Canvas({ authToken, logout, streamData, setDialog }) {
275256
// if x2 < x1*0.9 - it is probably a swipe right to left
276257
// if y2 < y1*0.9 - it is probably a swipe bottom to top
277258
// if y2 > y1*1.1 - it is probably a swipe top to bottom
278-
if (mouseEventsTimeDiff > 500 || coord2[0] > coord1[0] * 1.1 || coord2[0] < coord1[0] * 0.9 || coord2[1] < coord1[1] * 0.9 || coord2[1] > coord1[1] * 1.1) {
259+
if (mouseEventsTimeDiff > 500 && coord2[0] > coord1[0] * 1.1 || coord2[0] < coord1[0] * 0.9 || coord2[1] < coord1[1] * 0.9 || coord2[1] > coord1[1] * 1.1) {
279260
swipeCoordinates(authToken, logout, coord1, coord2, streamData, setDialog)
280261
} else if (mouseEventsTimeDiff < 500) {
281262
tapCoordinates(authToken, logout, coord1, streamData, setDialog)
@@ -314,20 +295,33 @@ function tapCoordinates(authToken, logout, pos, streamData, setDialog) {
314295
let x = pos[0]
315296
let y = pos[1]
316297

317-
// if the stream height
318-
if (streamData.canvasHeight != streamData.deviceY) {
319-
if (streamData.isPortrait) {
320-
x = (x / streamData.canvasWidth) * streamData.deviceX
321-
y = (y / streamData.canvasHeight) * streamData.deviceY
322-
} else {
323-
x = (x / streamData.canvasHeight) * streamData.deviceX
324-
y = (y / streamData.canvasWidth) * streamData.deviceY
298+
let finalX = (x / streamData.canvasWidth) * streamData.deviceX
299+
let finalY = (y / streamData.canvasHeight) * streamData.deviceY
300+
// If its portrait we keep the x and y as is
301+
if (!streamData.isPortrait) {
302+
// If its landscape
303+
// And its Android we still keep the x and y as is because for Android Appium does the coordinates are actually corresponding to the view
304+
// If we are in portrait X is X and Y is Y and when we are in landscape its the same
305+
if (streamData.device_os === 'android') {
306+
finalX = (x / streamData.canvasHeight) * streamData.deviceX
307+
finalY = (y / streamData.canvasWidth) * streamData.deviceY
308+
}
309+
if (streamData.device_os === 'ios') {
310+
if (streamData.uses_custom_wda) {
311+
finalX = streamData.deviceX - ((y / streamData.canvasHeight) * streamData.deviceX)
312+
finalY = (x / streamData.canvasWidth) * streamData.deviceY
313+
} else {
314+
// On normal WDA in landscape X corresponds to the canvas width but it is actually the device's height
315+
finalX = (x / streamData.canvasWidth) * streamData.deviceY
316+
// Y corresponds to the canvas height but it is actually the device's width
317+
finalY = (y / streamData.canvasHeight) * streamData.deviceX
318+
}
325319
}
326320
}
327321

328322
let jsonData = JSON.stringify({
329-
"x": x,
330-
"y": y
323+
"x": finalX,
324+
"y": finalY
331325
})
332326

333327
let deviceURL = `/device/${streamData.udid}`
@@ -353,15 +347,34 @@ function touchAndHoldCoordinates(authToken, logout, pos, streamData, setDialog)
353347
let x = pos[0]
354348
let y = pos[1]
355349

356-
// if the stream height
357-
if (streamData.canvasHeight != streamData.deviceY) {
358-
x = (x / streamData.canvasWidth) * streamData.deviceX
359-
y = (y / streamData.canvasHeight) * streamData.deviceY
350+
let finalX = (x / streamData.canvasWidth) * streamData.deviceX
351+
let finalY = (y / streamData.canvasHeight) * streamData.deviceY
352+
353+
// If its portrait we keep the x and y as is
354+
if (!streamData.isPortrait) {
355+
// If its landscape
356+
// And its Android we still keep the x and y as is because for Android Appium does the coordinates are actually corresponding to the view
357+
// If we are in portrait X is X and Y is Y and when we are in landscape its the same
358+
if (streamData.device_os === 'android') {
359+
finalX = (x / streamData.canvasHeight) * streamData.deviceX
360+
finalY = (y / streamData.canvasWidth) * streamData.deviceY
361+
}
362+
if (streamData.device_os === 'ios') {
363+
if (streamData.uses_custom_wda) {
364+
finalX = streamData.deviceX - ((y / streamData.canvasHeight) * streamData.deviceX)
365+
finalY = (x / streamData.canvasWidth) * streamData.deviceY
366+
} else {
367+
// On normal WDA in landscape X corresponds to the canvas width but it is actually the device's height
368+
finalX = (x / streamData.canvasWidth) * streamData.deviceY
369+
// Y corresponds to the canvas height but it is actually the device's width
370+
finalY = (y / streamData.canvasHeight) * streamData.deviceX
371+
}
372+
}
360373
}
361374

362375
let jsonData = JSON.stringify({
363-
"x": x,
364-
"y": y
376+
"x": finalX,
377+
"y": finalY
365378
})
366379

367380
let deviceURL = `/device/${streamData.udid}`
@@ -388,31 +401,66 @@ function swipeCoordinates(authToken, logout, coord1, coord2, streamData, setDial
388401
var secondCoordX = coord2[0]
389402
var secondCoordY = coord2[1]
390403

391-
// if the stream height
392-
if (streamData.canvasHeight != streamData.deviceY) {
393-
// If the device is landscape and is android
394-
// We need to switch the coordinate calculation
395-
// Divide by height for X and divide by width for Y
396-
if (streamData.device_os === 'android' && !streamData.isPortrait) {
397-
firstCoordX = (firstCoordX / streamData.canvasHeight) * streamData.deviceX
398-
firstCoordY = (firstCoordY / streamData.canvasWidth) * streamData.deviceY
399-
secondCoordX = (secondCoordX / streamData.canvasHeight) * streamData.deviceX
400-
secondCoordY = (secondCoordY / streamData.canvasWidth) * streamData.deviceY
404+
// Set up the portrait tap coordinates
405+
// We divide the current coordinate by the canvas size to get the ratio
406+
// Then we multiply by the actual device width or height to get the actual coordinates on the device that we should send
407+
let firstXFinal = (firstCoordX / streamData.canvasWidth) * streamData.deviceX
408+
let firstYFinal = (firstCoordY / streamData.canvasHeight) * streamData.deviceY
409+
let secondXFinal = (secondCoordX / streamData.canvasWidth) * streamData.deviceX
410+
let secondYFinal = (secondCoordY / streamData.canvasHeight) * streamData.deviceY
411+
412+
// If in landscape we need to do different recalculations
413+
if (!streamData.isPortrait) {
414+
// If the device is android we just reverse the calculations
415+
// For X we divide the coordinate by the canvas height to get the correct ratio
416+
// And for Y we divide the coordinate by the canvas width to get the correct ratio
417+
// Multiplication is the same as for portrait because Appium for Android follows some actual logic
418+
if (streamData.device_os === 'android') {
419+
firstXFinal = (firstCoordX / streamData.canvasHeight) * streamData.deviceX
420+
firstYFinal = (firstCoordY / streamData.canvasWidth) * streamData.deviceY
421+
secondXFinal = (secondCoordX / streamData.canvasHeight) * streamData.deviceX
422+
secondYFinal = (secondCoordY / streamData.canvasWidth) * streamData.deviceY
401423
} else {
402-
// If the device is not in landscape and is not android
403-
// Divide as usual - X by width and Y by height
404-
firstCoordX = (firstCoordX / streamData.canvasWidth) * streamData.deviceX
405-
firstCoordY = (firstCoordY / streamData.canvasHeight) * streamData.deviceY
406-
secondCoordX = (secondCoordX / streamData.canvasWidth) * streamData.deviceX
407-
secondCoordY = (secondCoordY / streamData.canvasHeight) * streamData.deviceY
424+
// For iOS its complete sh*t
425+
if (streamData.uses_custom_wda) {
426+
// NB: All calculations below are when the device is on its right side landscape(your left)
427+
// For custom WDA the 0 X coordinate is at the bottom of the canvas
428+
// And the 0 Y coordinate is at the left end of the canvas
429+
// This means that they kinda follow the portrait logic but inverted based on the side it is in landscape
430+
// Imagine a device that is X:Y=100:200
431+
// If you swipe vertically from bottom to the middle you are essentially swiping coordinates on the device X:0 and X: 50
432+
// And if you swipe horizontally from left to middle you are essentially swiping coordinates on the device Y: 0 and Y: 100
433+
// But on the canvas those are Y coordinates
434+
// And this is where it gets funky
435+
// For X we get the ratio from the canvas Y coordinate and the canvas height
436+
// And we multiply it by the device width because that is what it corresponds to
437+
// Then we subtract the value from the actual deviceX because the device width starts from the bottom of the canvas height
438+
firstXFinal = streamData.deviceX - ((firstCoordY / streamData.canvasHeight) * streamData.deviceX)
439+
// For Y we get the ratio from the canvas X coordinate and the canvas width
440+
// And we multiply it by the device height because that is what it corresponds to
441+
firstYFinal = (firstCoordX / streamData.canvasWidth) * streamData.deviceY
442+
// Same goes for the other two coordinates when the mouse is released
443+
secondXFinal = streamData.deviceX - ((secondCoordY / streamData.canvasHeight) * streamData.deviceX)
444+
secondYFinal = (secondCoordX / streamData.canvasWidth) * streamData.deviceY
445+
} else {
446+
// On normal WDA the X coordinates correspond to the device height
447+
// and the Y coordinates correspond to the device width
448+
// So we calculate ratio based on canvas dimensions and coordinates
449+
// But multiply by device height for X swipe coordinates and by device width for Y swipe coordinates
450+
firstXFinal = (firstCoordX / streamData.canvasWidth) * streamData.deviceY
451+
firstYFinal = (firstCoordY / streamData.canvasHeight) * streamData.deviceX
452+
secondXFinal = (secondCoordX / streamData.canvasWidth) * streamData.deviceY
453+
secondYFinal = (secondCoordY / streamData.canvasHeight) * streamData.deviceX
454+
}
408455
}
409456
}
457+
console.log('Swipe is ' + firstXFinal + ':' + firstYFinal + '-' + secondXFinal + ':' + secondYFinal)
410458

411459
let jsonData = JSON.stringify({
412-
"x": firstCoordX,
413-
"y": firstCoordY,
414-
"endX": secondCoordX,
415-
"endY": secondCoordY
460+
"x": firstXFinal,
461+
"y": firstYFinal,
462+
"endX": secondXFinal,
463+
"endY": secondYFinal
416464
})
417465

418466
let deviceURL = `/device/${streamData.udid}`

‎hub/gads-ui/src/components/DeviceSelection/DeviceSelection.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ export default function DeviceSelection() {
3737
CheckServerHealth()
3838

3939
// Use specific full address for local development, proxy does not seem to work okay
40-
// const evtSource = new EventSource(`http://192.168.68.109:10000/available-devices`);
40+
// const evtSource = new EventSource(`http://192.168.1.41:10000/available-devices`);
4141
const evtSource = new EventSource(`/available-devices`);
4242

4343
evtSource.onmessage = (message) => {

‎hub/gads-ui/src/services/axios.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@ import axios from 'axios'
22

33
export function GetAPIClient() {
44
const api = axios.create({
5-
// baseURL: `http://192.168.68.109:10000`
5+
// baseURL: `http://192.168.1.41:10000`
66
baseURL: ``
77
})
88

99
api.interceptors.request.use(
1010
async (config) => {
1111
const storedToken = localStorage.getItem('authToken')
1212

13-
if(storedToken) {
13+
if (storedToken) {
1414
config.headers['X-Auth-Token'] = `${storedToken}`
1515
}
1616

‎provider/router/appium.go

+52-34
Original file line numberDiff line numberDiff line change
@@ -109,44 +109,61 @@ func appiumTap(device *models.Device, x float64, y float64) (*http.Response, err
109109

110110
func appiumTouchAndHold(device *models.Device, x float64, y float64) (*http.Response, error) {
111111
// Generate the struct object for the Appium actions JSON request
112-
action := models.DevicePointerActions{
113-
Actions: []models.DevicePointerAction{
114-
{
115-
Type: "pointer",
116-
ID: "finger1",
117-
Parameters: models.DeviceActionParameters{
118-
PointerType: "touch",
119-
},
120-
Actions: []models.DeviceAction{
121-
{
122-
Type: "pointerMove",
123-
Duration: 0,
124-
X: x,
125-
Y: y,
126-
},
127-
{
128-
Type: "pointerDown",
129-
Button: 0,
130-
},
131-
{
132-
Type: "pause",
133-
Duration: 2000,
112+
if config.ProviderConfig.UseCustomWDA && device.OS == "ios" {
113+
requestBody := struct {
114+
X float64 `json:"x"`
115+
Y float64 `json:"y"`
116+
Delay float64 `json:"delay"`
117+
}{
118+
X: x,
119+
Y: y,
120+
Delay: 1,
121+
}
122+
actionJSON, err := json.MarshalIndent(requestBody, "", " ")
123+
if err != nil {
124+
return nil, err
125+
}
126+
return wdaRequest(device, http.MethodPost, "wda/touchAndHold", bytes.NewReader(actionJSON))
127+
} else {
128+
action := models.DevicePointerActions{
129+
Actions: []models.DevicePointerAction{
130+
{
131+
Type: "pointer",
132+
ID: "finger1",
133+
Parameters: models.DeviceActionParameters{
134+
PointerType: "touch",
134135
},
135-
{
136-
Type: "pointerUp",
137-
Duration: 0,
136+
Actions: []models.DeviceAction{
137+
{
138+
Type: "pointerMove",
139+
Duration: 0,
140+
X: x,
141+
Y: y,
142+
},
143+
{
144+
Type: "pointerDown",
145+
Button: 0,
146+
},
147+
{
148+
Type: "pause",
149+
Duration: 2000,
150+
},
151+
{
152+
Type: "pointerUp",
153+
Duration: 0,
154+
},
138155
},
139156
},
140157
},
141-
},
142-
}
158+
}
143159

144-
actionJSON, err := json.MarshalIndent(action, "", " ")
145-
if err != nil {
146-
return nil, err
147-
}
160+
actionJSON, err := json.MarshalIndent(action, "", " ")
161+
if err != nil {
162+
return nil, err
163+
}
148164

149-
return appiumRequest(device, http.MethodPost, "actions", bytes.NewReader(actionJSON))
165+
return appiumRequest(device, http.MethodPost, "actions", bytes.NewReader(actionJSON))
166+
}
150167
}
151168

152169
func appiumSwipe(device *models.Device, x, y, endX, endY float64) (*http.Response, error) {
@@ -187,8 +204,9 @@ func appiumSwipe(device *models.Device, x, y, endX, endY float64) (*http.Respons
187204
Y: y,
188205
},
189206
{
190-
Type: "pointerDown",
191-
Button: 0,
207+
Type: "pointerDown",
208+
Button: 0,
209+
Duration: 10,
192210
},
193211
{
194212
Type: "pointerMove",

0 commit comments

Comments
 (0)