Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,16 @@ Time taken to run the test.
Can be helpful to measure Time To Interactive of your app, if the test is checking app start for instance.
Average FPS
60 FPS
Frame Per Second. Your app should display 60 Frames Per Second to give an impression of fluidity. This number should be close to 60, otherwise it will seem laggy.
Frame Per Second. Your app should display 60 Frames Per Second to give an impression of fluidity. This number should be close to 60, otherwise it will seem laggy.

See

this video

for more details
Average CPU usage
83 %
An app might run at 60FPS but might be using too much processing power, so it's important to check CPU usage.
An app might run at high frame rates, such as 60 FPS or higher, but might be using too much processing power, so it's important to check CPU usage.
Depending on the device, this value can go up to
100% x number of cores
. For instance, a Samsung A10s has 4 cores, so the max value would be 400%.
Expand Down Expand Up @@ -855,7 +856,7 @@ exports[`flashlight measure interactive it displays measures: Web app with measu
<div
class="text-neutral-400 text-sm"
>
An app might run at 60FPS but might be using too much processing power, so it's important to check CPU usage.
An app might run at high frame rates, such as 60 FPS or higher, but might be using too much processing power, so it's important to check CPU usage.
<br />
Depending on the device, this value can go up to
<code>
Expand Down Expand Up @@ -3812,15 +3813,16 @@ Time taken to run the test.
Can be helpful to measure Time To Interactive of your app, if the test is checking app start for instance.
Average FPS
-
Frame Per Second. Your app should display 60 Frames Per Second to give an impression of fluidity. This number should be close to 60, otherwise it will seem laggy.
Frame Per Second. Your app should display 60 Frames Per Second to give an impression of fluidity. This number should be close to 60, otherwise it will seem laggy.

See

this video

for more details
Average CPU usage
-
An app might run at 60FPS but might be using too much processing power, so it's important to check CPU usage.
An app might run at high frame rates, such as 60 FPS or higher, but might be using too much processing power, so it's important to check CPU usage.
Depending on the device, this value can go up to
100% x number of cores
. For instance, a Samsung A10s has 4 cores, so the max value would be 400%.
Expand Down Expand Up @@ -4218,7 +4220,7 @@ exports[`flashlight measure interactive it displays measures: Web app with no me
<div
class="text-neutral-400 text-sm"
>
An app might run at 60FPS but might be using too much processing power, so it's important to check CPU usage.
An app might run at high frame rates, such as 60 FPS or higher, but might be using too much processing power, so it's important to check CPU usage.
<br />
Depending on the device, this value can go up to
<code>
Expand Down
17 changes: 10 additions & 7 deletions packages/commands/measure/src/server/ServerSocketConnectionApp.tsx
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import { PerformanceMeasurer } from "@perf-profiler/e2e";
import { Logger } from "@perf-profiler/logger";
import { profiler } from "@perf-profiler/profiler";
import { Measure } from "@perf-profiler/types";
import React, { useCallback, useEffect } from "react";
import { HostAndPortInfo } from "./components/HostAndPortInfo";
import { SocketType } from "./socket/socketInterface";
import { SocketType, SocketEvents } from "./socket/socketInterface";
import { useSocketState, updateMeasuresReducer, addNewResultReducer } from "./socket/socketState";
import { useBundleIdControls } from "./useBundleIdControls";
import { useLogSocketEvents } from "../common/useLogSocketEvents";
Expand Down Expand Up @@ -33,9 +34,11 @@ export const ServerSocketConnectionApp = ({ socket, url }: { socket: SocketType;
)
);

socket.on("start", async () => {
socket.on(SocketEvents.START, async () => {
const refreshRate = profiler.detectDeviceRefreshRate();
setState({
isMeasuring: true,
refreshRate,
});

if (!state.bundleId) {
Expand All @@ -55,19 +58,19 @@ export const ServerSocketConnectionApp = ({ socket, url }: { socket: SocketType;
);
});

socket.on("stop", stop);
socket.on(SocketEvents.STOP, stop);

socket.on("reset", () => {
socket.on(SocketEvents.RESET, () => {
stop();
setState({
results: [],
});
});

return () => {
socket.removeAllListeners("start");
socket.removeAllListeners("stop");
socket.removeAllListeners("reset");
socket.removeAllListeners(SocketEvents.START);
socket.removeAllListeners(SocketEvents.STOP);
socket.removeAllListeners(SocketEvents.RESET);
};
}, [setState, socket, state.bundleId, stop]);

Expand Down
16 changes: 16 additions & 0 deletions packages/commands/measure/src/server/socket/socketInterface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ export interface SocketData {
isMeasuring: boolean;
bundleId: string | null;
results: TestCaseResult[];
refreshRate: number;
}

export interface ServerToClientEvents {
Expand All @@ -18,6 +19,7 @@ export interface ClientToServerEvents {
reset: () => void;
autodetectBundleId: () => void;
setBundleId: (bundleId: string) => void;
autodetectRefreshRate: () => void;
}

interface InterServerEvents {
Expand All @@ -37,3 +39,17 @@ export type SocketType = Socket<
InterServerEvents,
SocketData
>;

export enum SocketEvents {
START = "start",
STOP = "stop",
RESET = "reset",
AUTODETECT_BUNDLE_ID = "autodetectBundleId",
SET_BUNDLE_ID = "setBundleId",
AUTODETECT_REFRESH_RATE = "autodetectRefreshRate",
UPDATE_STATE = "updateState",
SEND_ERROR = "sendError",
PING = "ping",
CONNECT = "connect",
DISCONNECT = "disconnect",
}
8 changes: 6 additions & 2 deletions packages/commands/measure/src/server/socket/socketState.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
import { Measure, POLLING_INTERVAL } from "@perf-profiler/types";
import { useState, useEffect } from "react";
import { SocketType, SocketData } from "./socketInterface";
import { SocketType, SocketData, SocketEvents } from "./socketInterface";

export const useSocketState = (socket: SocketType) => {
const [state, _setState] = useState<SocketData>({
isMeasuring: false,
bundleId: null,
results: [],
refreshRate: 60,
});

const setState = (
Expand All @@ -23,7 +24,7 @@ export const useSocketState = (socket: SocketType) => {
};

useEffect(() => {
socket.emit("updateState", state);
socket.emit(SocketEvents.UPDATE_STATE, state);
}, [state, socket]);

return [state, setState] as const;
Expand Down Expand Up @@ -54,6 +55,9 @@ export const addNewResultReducer = (state: SocketData, name: string): SocketData
name,
iterations: [],
status: "SUCCESS",
specs: {
refreshRate: state.refreshRate,
},
},
],
});
25 changes: 19 additions & 6 deletions packages/commands/measure/src/server/useBundleIdControls.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
import { profiler } from "@perf-profiler/profiler";
import { useEffect } from "react";
import { SocketType, SocketData } from "./socket/socketInterface";
import { SocketType, SocketData, SocketEvents } from "./socket/socketInterface";

export const useBundleIdControls = (
socket: SocketType,
setState: (state: Partial<SocketData>) => void,
stop: () => void
) => {
useEffect(() => {
socket.on("setBundleId", (bundleId) => {
socket.on(SocketEvents.SET_BUNDLE_ID, (bundleId) => {
setState({
bundleId,
});
});

socket.on("autodetectBundleId", () => {
socket.on(SocketEvents.AUTODETECT_BUNDLE_ID, () => {
stop();

try {
Expand All @@ -23,13 +23,26 @@ export const useBundleIdControls = (
bundleId,
});
} catch (error) {
socket.emit("sendError", error instanceof Error ? error.message : "unknown error");
socket.emit(
SocketEvents.SEND_ERROR,
error instanceof Error ? error.message : "unknown error"
);
}
});

socket.on(SocketEvents.AUTODETECT_REFRESH_RATE, () => {
stop();

const refreshRate = profiler.detectDeviceRefreshRate();
setState({
refreshRate,
});
});

return () => {
socket.removeAllListeners("setBundleId");
socket.removeAllListeners("autodetectBundleId");
socket.removeAllListeners(SocketEvents.SET_BUNDLE_ID);
socket.removeAllListeners(SocketEvents.AUTODETECT_BUNDLE_ID);
socket.removeAllListeners(SocketEvents.AUTODETECT_REFRESH_RATE);
};
}, [setState, socket, stop]);
};
13 changes: 7 additions & 6 deletions packages/commands/measure/src/webapp/components/SocketState.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import Button from "@mui/material/Button";
import { Logger } from "@perf-profiler/logger";
import { socket } from "../socket";
import { useLogSocketEvents } from "../../common/useLogSocketEvents";
import { SocketEvents } from "../../server/socket/socketInterface";

const useSocketState = (onError: (error: string) => void) => {
useLogSocketEvents(socket);
Expand All @@ -28,14 +29,14 @@ const useSocketState = (onError: (error: string) => void) => {
}
}

socket.on("connect", onConnect);
socket.on("disconnect", onDisconnect);
socket.on("sendError", onError);
socket.on(SocketEvents.CONNECT, onConnect);
socket.on(SocketEvents.DISCONNECT, onDisconnect);
socket.on(SocketEvents.SEND_ERROR, onError);

return () => {
socket.off("connect", onConnect);
socket.off("disconnect", onDisconnect);
socket.off("sendError", onError);
socket.off(SocketEvents.CONNECT, onConnect);
socket.off(SocketEvents.DISCONNECT, onDisconnect);
socket.off(SocketEvents.SEND_ERROR, onError);
};
}, [onError]);

Expand Down
8 changes: 6 additions & 2 deletions packages/commands/measure/src/webapp/socket.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
import { io, Socket } from "socket.io-client";
import { ServerToClientEvents, ClientToServerEvents } from "../server/socket/socketInterface";
import {
ServerToClientEvents,
ClientToServerEvents,
SocketEvents,
} from "../server/socket/socketInterface";

export const socket: Socket<ServerToClientEvents, ClientToServerEvents> = io(
window.__FLASHLIGHT_DATA__.socketServerUrl
);

socket.on("disconnect", () => socket.close());
socket.on(SocketEvents.DISCONNECT, () => socket.close());
18 changes: 10 additions & 8 deletions packages/commands/measure/src/webapp/useMeasures.ts
Original file line number Diff line number Diff line change
@@ -1,36 +1,38 @@
import { useEffect, useState } from "react";
import type { SocketData } from "../server/socket/socketInterface";
import { SocketData, SocketEvents } from "../server/socket/socketInterface";
import { socket } from "./socket";

export const useMeasures = () => {
const [state, setState] = useState<SocketData>();

useEffect(() => {
socket.on("updateState", setState);
socket.on(SocketEvents.UPDATE_STATE, setState);

return () => {
socket.off("updateState", setState);
socket.off(SocketEvents.UPDATE_STATE, setState);
};
}, []);

return {
bundleId: state?.bundleId ?? null,
refreshRate: state?.refreshRate ?? 60,
autodetect: () => {
socket.emit("autodetectBundleId");
socket.emit(SocketEvents.AUTODETECT_BUNDLE_ID);
socket.emit(SocketEvents.AUTODETECT_REFRESH_RATE);
},
setBundleId: (bundleId: string) => {
socket.emit("setBundleId", bundleId);
socket.emit(SocketEvents.SET_BUNDLE_ID, bundleId);
},
results: state?.results ?? [],
isMeasuring: state?.isMeasuring ?? false,
start: () => {
socket.emit("start");
socket.emit(SocketEvents.START);
},
stop: () => {
socket.emit("stop");
socket.emit(SocketEvents.STOP);
},
reset: () => {
socket.emit("reset");
socket.emit(SocketEvents.RESET);
},
};
};
4 changes: 4 additions & 0 deletions packages/core/reporter/src/reporting/Report.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,4 +117,8 @@ export class Report {
threads: getThreadsStats(iterations),
};
}

public getRefreshRate() {
return this.result.specs?.refreshRate ?? 60;
}
}
2 changes: 1 addition & 1 deletion packages/core/reporter/src/reporting/getScore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export const getScore = (result: AveragedTestCaseResult) => {
const scores = [cpuScore];

if (averageUIFPS !== undefined) {
const fpsScore = (averageUIFPS * 100) / 60;
const fpsScore = (averageUIFPS * 100) / (result?.specs?.refreshRate ?? 60);
scores.push(fpsScore);
}

Expand Down
7 changes: 7 additions & 0 deletions packages/core/types/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ export interface TestCaseResult {
status: TestCaseResultStatus;
iterations: TestCaseIterationResult[];
type?: TestCaseResultType;
specs?: DeviceSpecs;
}

export interface AveragedTestCaseResult {
Expand All @@ -49,6 +50,7 @@ export interface AveragedTestCaseResult {
average: TestCaseIterationResult;
averageHighCpuUsage: { [processName: string]: number };
type?: TestCaseResultType;
specs?: DeviceSpecs;
}

// Shouldn't really be here but @perf-profiler/types is imported by everyone and doesn't contain any logic
Expand Down Expand Up @@ -97,4 +99,9 @@ export interface Profiler {
cleanup: () => void;
getScreenRecorder: (videoPath: string) => ScreenRecorder | undefined;
stopApp: (bundleId: string) => Promise<void>;
detectDeviceRefreshRate: () => number;
}

export interface DeviceSpecs {
refreshRate: number;
}
Loading