Add Game Overlay

README.md

@@ -30,7 +30,8 @@ who wants to communicate with text-to-speech and for that reason, you can see Iz
 - [x] Multiple text-to-speech engines (Amazon Polly, Microsoft Azure, Google Cloud, IBM Watson,
       etc..) - [see list](#available-engines)
 - [x] Multiple voices (per engine)
-- [x] Supports custom text-to-speech engines [(see example)](https://github.com/nature-heart-software/izabela/tree/dev/examples/custom-engine-api)
+- [x] Supports custom text-to-speech
+      engines [(see example)](https://github.com/nature-heart-software/izabela/tree/dev/examples/custom-engine-api)
 - [x] Multiple audio outputs
 - [x] Supports audio inputs for speech-to-text-to-speech
 - [x] Keyboard shortcuts for quick messages
@@ -75,7 +76,8 @@ the [Available engines](#available-engines) section.
 > **Note:** The Izabela engine is only available to "Supporter" members or above
 > on [Ko-fi](https://ko-fi.com/woowee/tiers).
 
-If you prefer configuring your credentials yourself, you can do so by following the guides on the [Wiki](https://github.com/nature-heart-software/izabela/wiki)
+If you prefer configuring your credentials yourself, you can do so by following the guides on
+the [Wiki](https://github.com/nature-heart-software/izabela/wiki)
 page.
 
 ## Available engines
@@ -112,3 +114,4 @@ The project requires the following in order to run:
 - Visual Studio Build Tools 2019 or higher
   - Desktop development with C++
   - Node.js build tools
+- CMake

apps/app/package.json

@@ -5,14 +5,20 @@
   "author": "Nature Heart Software",
   "dependencies": {
     "@apps/app-server": "0.0.0",
+    "@nut-tree-fork/nut-js": "^4.2.1",
+    "@packages/electron-game-overlay": "0.0.0",
     "@packages/native-keymap": "3.3.2",
     "@packages/win-mouse": "2.0.1",
     "ffi-napi": "^4.0.3",
+    "micromatch": "^4.0.5",
     "node-abi": "^3.40.0",
     "node-global-key-listener": "^0.1.1",
     "node-record-lpcm16": "1.0.1",
     "sudo-prompt": "^9.2.1",
-    "systeminformation": "^5.17.3"
+    "systeminformation": "^5.17.3",
+    "win-control": "^0.3.1",
+    "windows-tlist": "^0.1.3",
+    "wql-process-monitor": "3.0.1"
   },
   "devDependencies": {
     "@packages/animalese": "0.0.0",

apps/app/resources/detect-game.js

@@ -0,0 +1,44 @@
+const wqlImport = import('wql-process-monitor')
+
+function send(...args) {
+  process.send(...args)
+}
+
+wqlImport
+  .then((wql) => {
+    const { subscribe } = wql
+    return subscribe({
+      creation: true,
+      deletion: true,
+      bin: {
+        filter: ['cmd.exe', 'tlist.exe', 'conhost.exe'],
+      },
+    })
+  })
+  .then((processMonitor) => {
+    // require('windows-tlist').getProcessInfo(pid).then(console.log)
+    processMonitor.on('creation', ([process, pid, filepath, user]) => {
+      send({
+        type: 'process-creation',
+        payload: {
+          process,
+          pid,
+          filepath,
+          user,
+        },
+      })
+    })
+    processMonitor.on('deletion', ([process, pid, filepath, user]) => {
+      send({
+        type: 'process-deletion',
+        payload: {
+          process,
+          pid,
+          filepath,
+          user,
+        },
+      })
+    })
+  })
+
+setInterval(() => {}, 1000 * 60 * 60)

apps/app/src/consts.ts

@@ -1 +1,2 @@
 export const DEFAULT_LANGUAGE_CODE = 'en-US'
+export const isGameOverlay = new URLSearchParams(location.search).has('game-overlay')

apps/app/src/electron/app.ts

@@ -1,13 +1,16 @@
 import { app, BrowserWindow, protocol } from 'electron'
-import installExtension, { VUEJS3_DEVTOOLS } from 'electron-devtools-installer'
+import installExtension, { VUEJS_DEVTOOLS } from 'electron-devtools-installer'
 import path from 'path'
 import server from '@apps/app-server'
 import { electronPiniaPlugin } from '@packages/electron-pinia/main'
 import { createApp, h } from 'vue'
 import { createPinia } from 'pinia'
 import createTray from '@/teams/tray/electron-tray'
 import ElectronWindowManager from '@/modules/electron-window-manager'
-import { createMessengerWindow } from '@/teams/messenger/electron/background'
+import {
+  createMessengerGameOverlayWindow,
+  createMessengerWindow,
+} from '@/teams/messenger/electron/background'
 import { createSpeechWorkerWindow } from '@/teams/speech-worker/electron/background'
 import { bridgeModules } from '@/electron/bridge'
 import registerElectronStartup from '@/modules/electron-startup/register'
@@ -18,6 +21,9 @@ import registerElectronKeybinding from '@/modules/electron-keybinding/register'
 import registerElectronCache from '@/modules/electron-cache/register'
 import { destroyWinMouse } from '@/modules/node-mouse'
 import { createOverlayWindow } from '@/teams/overlay/electron/background'
+import './game-overlay'
+import gameOverlay from '@/electron/game-overlay.ts'
+import { gkl } from '@/modules/electron-keybinding/utils.ts'
 
 const App = () => {
   const isDevelopment = import.meta.env.DEV
@@ -26,18 +32,22 @@ const App = () => {
       .whenReady()
       .then(async () =>
         Promise.all([
-          await ElectronWindowManager.registerInstance(
+          ElectronWindowManager.registerInstance(
             'messenger',
             createMessengerWindow,
           ),
-          await ElectronWindowManager.registerInstance(
+          ElectronWindowManager.registerInstance(
             'overlay',
             createOverlayWindow,
           ),
-          await ElectronWindowManager.registerInstance(
+          ElectronWindowManager.registerInstance(
             'speech-worker',
             createSpeechWorkerWindow,
           ),
+          ElectronWindowManager.registerInstance(
+            'messenger-game-overlay',
+            createMessengerGameOverlayWindow,
+          ),
         ]),
       )
 
@@ -47,6 +57,9 @@ const App = () => {
     )
   }
 
+  const startGameOverlay = async () =>
+    app.whenReady().then(() => gameOverlay.start())
+
   const startAppServer = async () =>
     app.whenReady().then(async () =>
       server.start({
@@ -83,6 +96,8 @@ const App = () => {
   }
 
   const onQuit = () => {
+    gkl?.kill()
+    gameOverlay.quit()
     /* fixes app.quit(): https://stackoverflow.com/a/75369483 */
     ElectronWindowManager.getInstances().forEach(({ window }) => {
       window.removeAllListeners()
@@ -107,7 +122,9 @@ const App = () => {
     app.on('ready', async () => {
       if (isDevelopment) {
         try {
-          await installExtension(VUEJS3_DEVTOOLS)
+          await ((installExtension as any).default as typeof installExtension)(
+            VUEJS_DEVTOOLS,
+          )
         } catch (e) {
           console.error(
             'Vue Devtools failed to install:',
@@ -148,6 +165,15 @@ const App = () => {
         handleQuit(true)
       }
     })
+
+    app.on('web-contents-created', (_, webContents) => {
+      webContents.on('preload-error', (_, preloadPath, error) => {
+        console.error(
+          `Preload script error:\nPath: ${preloadPath}\nError:`,
+          error,
+        )
+      })
+    })
   }
 
   function start() {
@@ -159,6 +185,7 @@ const App = () => {
       exec('Register startup', () => registerElectronStartup()),
       exec('Register debug', () => registerElectronDebug()),
       exec('Bridge modules', () => bridgeModules()),
+      exec('Start Game Overlay', () => startGameOverlay()),
       exec('Create tray', () => createTray()),
       exec('Create windows', () => createWindows()),
       exec('Start server', () => startAppServer()),

apps/app/src/electron/bridge.ts

@@ -2,13 +2,13 @@ import { bridge } from '@packages/electron-bridger'
 import electronDialog from '@/modules/electron-dialog'
 import electronFilesystem from '@/modules/electron-filesystem'
 import electronMessengerWindow from '@/teams/messenger/modules/electron-messenger-window'
+import electronOverlayWindow from '@/teams/overlay/modules/electron-overlay-window'
 import electronSpeechWorkerWindow from '@/teams/speech-worker/modules/electron-speech-worker-window'
 import electronKeybinding from '@/modules/electron-keybinding'
 import electronDisplay from '@/modules/electron-display'
 import electronResources from '@/modules/electron-resources'
 import electronSi from '@/modules/electron-si'
 import electronTranslation from '@/modules/electron-translation'
-import electronOverlayWindow from '@/teams/overlay/modules/electron-overlay-window'
 
 export const bridgeModules = () =>
   bridge.register([

apps/app/src/electron/events/main.ts

@@ -1,14 +1,28 @@
 import { processes } from '@/types/electron'
 import { ipcMain } from 'electron-postman'
 
-export const onIPCProcessError = (callback: (error: Error, process: string) => any) => {
+export const onIPCProcessError = (
+  callback: (error: Error, process: string) => any,
+) => {
   processes.forEach((process) => {
     ipcMain.on(process, 'error', (payload: { name: string; message: string }) =>
       callback(payload, process),
     )
   })
 }
 
+export const onIPCGameOverlayStartIntercept = (callback: () => any) => {
+  processes.forEach((process) => {
+    ipcMain.on(process, 'game-overlay-start-intercept', () => callback())
+  })
+}
+
+export const onIPCGameOverlayStopIntercept = (callback: () => any) => {
+  processes.forEach((process) => {
+    ipcMain.on(process, 'game-overlay-stop-intercept', () => callback())
+  })
+}
+
 export const emitIPCCancelCurrentMessage = () => {
   ipcMain.sendTo('speech-worker', 'cancel-current-message')
 }
@@ -21,6 +35,16 @@ export const emitIPCOverlayInputCharacter = (character: string) => {
   ipcMain.sendTo('overlay', 'overlay-input-character', character)
 }
 
-export const emitIPCOverlayInputCommand = (command: string, args: any[] = []) => {
+export const emitIPCOverlayInputCommand = (
+  command: string,
+  args: any[] = [],
+) => {
   ipcMain.sendTo('overlay', 'overlay-input-command', [command, ...args])
 }
+
+export const emitIPCGameOverlayResize = (size: {
+  width: number
+  height: number
+}) => {
+  ipcMain.sendTo('messenger-game-overlay', 'resize', size)
+}

apps/app/src/electron/events/renderer.ts

@@ -1,14 +1,25 @@
-import { processes } from '@/types/electron'
+import { mainProcess, processes } from '@/types/electron'
 import { IzabelaMessage } from '@/modules/izabela/types'
 
 const { ipc } = window
 
-export const emitIPCProcessError = (payload: { name: string; message: string }) => {
+export const emitIPCProcessError = (payload: {
+  name: string
+  message: string
+}) => {
   processes.forEach((process) => {
     ipc.sendTo(process, 'error', payload)
   })
 }
 
+export const emitIPCGameOverlayStartIntercept = () => {
+  ipc.sendTo(mainProcess, 'game-overlay-start-intercept')
+}
+
+export const emitIPCGameOverlayStopIntercept = () => {
+  ipc.sendTo(mainProcess, 'game-overlay-stop-intercept')
+}
+
 type IPCSayPayload = string | IzabelaMessage
 
 export const onIPCSay = (callback: (payload: IPCSayPayload) => any) => {
@@ -44,3 +55,11 @@ export const onIPCOverlayInputCommand = (callback: (args: any[]) => void) => {
     ipc.on(process, 'overlay-input-command', callback)
   })
 }
+
+export const onIPCGameOverlayResize = (
+  callback: (size: { width: number; height: number }) => void,
+) => {
+  processes.forEach((process) => {
+    ipc.on(process, 'resize', callback)
+  })
+}