feat: add demo of GPT vision capabilities based on video stream

assets/app.js

@@ -4,4 +4,5 @@ import './styles/app.css';
 import './styles/audio.css';
 import './styles/blog.css';
 import './styles/youtube.css';
+import './styles/video.css';
 import './styles/wikipedia.css';

assets/controllers/video_controller.js

@@ -0,0 +1,43 @@
+import {Controller} from '@hotwired/stimulus';
+import {getComponent} from '@symfony/ux-live-component';
+
+export default class extends Controller {
+    async initialize() {
+        this.component = await getComponent(this.element);
+
+        this.video = document.getElementById('videoFeed');
+        this.canvas = document.getElementById('canvas');
+
+        const input = document.getElementById('chat-message');
+        input.addEventListener('keypress', (event) => {
+            if (event.key === 'Enter') {
+                this.submitMessage();
+            }
+        });
+        input.focus();
+
+        const submitButton = document.getElementById('chat-submit');
+        submitButton.addEventListener('click', (event) => {
+            this.submitMessage();
+        });
+
+        this.video.srcObject = await navigator.mediaDevices.getUserMedia({video: true, audio: false});
+    };
+
+    submitMessage() {
+        const input = document.getElementById('chat-message');
+        const instruction = input.value;
+        const image = this.captureImage();
+
+        this.component.action('submit', { instruction, image });
+        input.value = '';
+    }
+
+    captureImage() {
+        this.canvas.width = this.video.videoWidth;
+        this.canvas.height = this.video.videoHeight;
+        const context = this.canvas.getContext('2d');
+        context.drawImage(this.video, 0, 0, this.canvas.width, this.canvas.height);
+        return this.canvas.toDataURL('image/jpeg', 0.8);
+    }
+}

assets/styles/video.css

@@ -0,0 +1,27 @@
+.video {
+    body&, .card-img-top {
+        background: #26931e;
+        background: linear-gradient(0deg, #186361 0%, #26931e 100%);
+    }
+
+    .card-img-top {
+        color: #ffffff;
+    }
+
+    &.chat {
+        #chat-submit {
+            &:hover {
+                background: #186361;
+                border-color: #186361;
+            }
+        }
+    }
+
+    footer {
+        color: #ffffff;
+
+        a {
+            color: #ffffff;
+        }
+    }
+}

config/routes.yaml

@@ -25,6 +25,13 @@ youtube:
         template:  'chat.html.twig'
         context: { chat: 'youtube' }
 
+video:
+    path: '/video'
+    controller: 'Symfony\Bundle\FrameworkBundle\Controller\TemplateController'
+    defaults:
+        template:  'chat.html.twig'
+        context: { chat: 'video' }
+
 wikipedia:
     path: '/wikipedia'
     controller: 'Symfony\Bundle\FrameworkBundle\Controller\TemplateController'

src/Video/TwigComponent.php

@@ -0,0 +1,55 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Video;
+
+use PhpLlm\LlmChain\Bridge\OpenAI\GPT;
+use PhpLlm\LlmChain\Model\Message\Content\Image;
+use PhpLlm\LlmChain\Model\Message\Message;
+use PhpLlm\LlmChain\Model\Message\MessageBag;
+use PhpLlm\LlmChain\Model\Response\AsyncResponse;
+use PhpLlm\LlmChain\Model\Response\TextResponse;
+use PhpLlm\LlmChain\PlatformInterface;
+use Symfony\UX\LiveComponent\Attribute\AsLiveComponent;
+use Symfony\UX\LiveComponent\Attribute\LiveAction;
+use Symfony\UX\LiveComponent\Attribute\LiveArg;
+use Symfony\UX\LiveComponent\Attribute\LiveProp;
+use Symfony\UX\LiveComponent\DefaultActionTrait;
+
+#[AsLiveComponent('video')]
+final class TwigComponent
+{
+    use DefaultActionTrait;
+
+    #[LiveProp]
+    public string $caption = 'Please define an instruction and hit submit.';
+
+    public function __construct(
+        private readonly PlatformInterface $platform,
+    ) {
+    }
+
+    #[LiveAction]
+    public function submit(#[LiveArg] string $instruction, #[LiveArg] string $image): void
+    {
+        $messageBag = new MessageBag(
+            Message::forSystem(<<<PROMPT
+                You are a video captioning assistant. You are provided with a video frame and an instruction.
+                You must generate a caption or answer based on the provided video frame and the user's instruction.
+                You are not in a conversation with the user and there will be no follow-up questions or messages.
+                PROMPT),
+            Message::ofUser($instruction, Image::fromDataUrl($image))
+        );
+
+        $response = $this->platform->request(new GPT(GPT::GPT_4O_MINI), $messageBag, [
+            'max_tokens' => 100,
+        ]);
+
+        assert($response instanceof AsyncResponse);
+        $response = $response->unwrap();
+        assert($response instanceof TextResponse);
+
+        $this->caption = $response->getContent();
+    }
+}

templates/base.html.twig

@@ -34,6 +34,9 @@
                         <li class="nav-item">
                             <a class="nav-link" href="{{ path('audio') }}">{{ ux_icon('iconoir:microphone-solid', { height: '20px', width: '20px' }) }} Audio Bot</a>
                         </li>
+                        <li class="nav-item">
+                            <a class="nav-link" href="{{ path('video') }}">{{ ux_icon('tabler:video-filled', { height: '20px', width: '20px' }) }} Video Bot</a>
+                        </li>
                         <li class="nav-item"><span class="nav-link">|</span></li>
                         <li class="nav-item">
                             <a class="nav-link" href="https://github.com/php-llm/symfony-demo" target="_blank">{{ ux_icon('mdi:github', { height: '20px', width: '20px' }) }} GitHub</a>

templates/components/video.html.twig

@@ -0,0 +1,23 @@
+{% import "_message.html.twig" as message %}
+
+<div class="card mx-auto shadow-lg" {{ attributes.defaults(stimulus_controller('video')) }}>
+    <div class="card-header p-2">
+        {{ ux_icon('tabler:video-filled', { height: '32px', width: '32px' }) }}
+        <strong class="ms-1 pt-1 d-inline-block">Video Bot</strong>
+    </div>
+    <div id="chat-body" class="card-body p-2 overflow-auto">
+        <div id="welcome" class="text-center mt-5 p-4 bg-white rounded-5 shadow-sm w-75 mx-auto">
+            <div class="mb-2">
+                <video id="videoFeed" autoplay playsinline></video>
+            </div>
+            <canvas id="canvas" class="d-none"></canvas>
+            <i class="text-muted">{{ this.caption }}</i>
+        </div>
+    </div>
+    <div class="card-footer p-2">
+        <div class="input-group">
+            <input id="chat-message" type="text" class="form-control border-0" placeholder="What do you see?">
+            <button id="chat-submit" class="btn btn-outline-secondary border-0" type="button">{{ ux_icon('mingcute:send-fill', { height: '25px', width: '25px' }) }} Submit</button>
+        </div>
+    </div>
+</div>

templates/index.html.twig

@@ -13,7 +13,7 @@
         </p>
         <h3 class="text-dark">Examples</h3>
         <div class="row">
-            <div class="col-md-3">
+            <div class="col-md-4">
                 <div class="card blog bg-body shadow-sm">
                     <div class="card-img-top py-2">
                         {{ ux_icon('mdi:symfony', { height: '150px', width: '150px' }) }}
@@ -32,7 +32,7 @@
                     {% endif %}
                 </div>
             </div>
-            <div class="col-md-3">
+            <div class="col-md-4">
                 <div class="card youtube bg-body shadow-sm">
                     <div class="card-img-top py-2">
                         {{ ux_icon('bi:youtube', { height: '150px', width: '150px' }) }}
@@ -51,7 +51,7 @@
                     {% endif %}
                 </div>
             </div>
-            <div class="col-md-3">
+            <div class="col-md-4">
                 <div class="card wikipedia bg-body shadow-sm">
                     <div class="card-img-top py-2">
                         {{ ux_icon('mdi:wikipedia', { height: '150px', width: '150px' }) }}
@@ -70,14 +70,17 @@
                     {% endif %}
                 </div>
             </div>
-            <div class="col-md-3">
+        </div>
+        <div class="row mt-4">
+            <div class="col-md-2"></div>
+            <div class="col-md-4">
                 <div class="card audio bg-body shadow-sm">
                     <div class="card-img-top py-2">
                         {{ ux_icon('iconoir:microphone-solid', { height: '150px', width: '150px' }) }}
                     </div>
                     <div class="card-body">
                         <h5 class="card-title">Audio Bot</h5>
-                        <p class="card-text">Simple demonstration of text-to-speech with Whisper in combination with GPT.</p>
+                        <p class="card-text">Simple demonstration of speech-to-text with Whisper in combination with GPT.</p>
                         <a href="{{ path('audio') }}" class="btn btn-outline-dark d-block">Try Audio Bot</a>
                     </div>
                     {# Profiler route only available in dev #}
@@ -89,6 +92,26 @@
                     {% endif %}
                 </div>
             </div>
+            <div class="col-md-4">
+                <div class="card video bg-body shadow-sm">
+                    <div class="card-img-top py-2">
+                        {{ ux_icon('tabler:video-filled', { height: '150px', width: '150px' }) }}
+                    </div>
+                    <div class="card-body">
+                        <h5 class="card-title">Video Bot</h5>
+                        <p class="card-text">Simple demonstration of vision capabilities of GPT in combination with your webcam.</p>
+                        <a href="{{ path('video') }}" class="btn btn-outline-dark d-block">Try Video Bot</a>
+                    </div>
+                    {# Profiler route only available in dev #}
+                    {% if 'dev' == app.environment %}
+                        <div class="card-footer">
+                            {{ ux_icon('solar:code-linear', { height: '20px', width: '20px' }) }}
+                            <a href="{{ path('_profiler_open_file', { file: 'src/Video/TwigComponent.php', line: 37 }) }}">See Implementation</a>
+                        </div>
+                    {% endif %}
+                </div>
+            </div>
+            <div class="col-md-2"></div>
         </div>
     </div>
 {% endblock %}

tests/SmokeTest.php

@@ -21,7 +21,7 @@ public function testIndex(): void
 
         self::assertResponseIsSuccessful();
         self::assertSelectorTextSame('h1', 'Welcome to the LLM Chain Demo');
-        self::assertSelectorCount(4, '.card');
+        self::assertSelectorCount(5, '.card');
     }
 
     #[DataProvider('provideChats')]