Skip to content

Commit f2aa2db

Browse files
chr-hertelchr-hertel
committed
feat: add demo of GPT vision capabilities based on video stream
1 parent 0900c45 commit f2aa2db

File tree

10 files changed

+188
-5
lines changed

10 files changed

+188
-5
lines changed

‎assets/app.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ import './styles/app.css';
44
import './styles/audio.css';
55
import './styles/blog.css';
66
import './styles/youtube.css';
7+
import './styles/video.css';
78
import './styles/wikipedia.css';

‎assets/controllers/video_controller.js

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import {Controller} from '@hotwired/stimulus';
2+
import {getComponent} from '@symfony/ux-live-component';
3+
4+
export default class extends Controller {
5+
async initialize() {
6+
this.component = await getComponent(this.element);
7+
8+
this.video = document.getElementById('videoFeed');
9+
this.canvas = document.getElementById('canvas');
10+
11+
const input = document.getElementById('chat-message');
12+
input.addEventListener('keypress', (event) => {
13+
if (event.key === 'Enter') {
14+
this.submitMessage();
15+
}
16+
});
17+
input.focus();
18+
19+
const submitButton = document.getElementById('chat-submit');
20+
submitButton.addEventListener('click', (event) => {
21+
this.submitMessage();
22+
});
23+
24+
this.video.srcObject = await navigator.mediaDevices.getUserMedia({video: true, audio: false});
25+
};
26+
27+
submitMessage() {
28+
const input = document.getElementById('chat-message');
29+
const instruction = input.value;
30+
const image = this.captureImage();
31+
32+
this.component.action('submit', { instruction, image });
33+
input.value = '';
34+
}
35+
36+
captureImage() {
37+
this.canvas.width = this.video.videoWidth;
38+
this.canvas.height = this.video.videoHeight;
39+
const context = this.canvas.getContext('2d');
40+
context.drawImage(this.video, 0, 0, this.canvas.width, this.canvas.height);
41+
return this.canvas.toDataURL('image/jpeg', 0.8);
42+
}
43+
}

‎assets/icons/tabler/video-filled.svg

Lines changed: 1 addition & 0 deletions
Loading

‎assets/styles/video.css

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
.video {
2+
body&, .card-img-top {
3+
background: #26931e;
4+
background: linear-gradient(0deg, #186361 0%, #26931e 100%);
5+
}
6+
7+
.card-img-top {
8+
color: #ffffff;
9+
}
10+
11+
&.chat {
12+
#chat-submit {
13+
&:hover {
14+
background: #186361;
15+
border-color: #186361;
16+
}
17+
}
18+
}
19+
20+
footer {
21+
color: #ffffff;
22+
23+
a {
24+
color: #ffffff;
25+
}
26+
}
27+
}

‎config/routes.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@ youtube:
2525
template: 'chat.html.twig'
2626
context: { chat: 'youtube' }
2727

28+
video:
29+
path: '/video'
30+
controller: 'Symfony\Bundle\FrameworkBundle\Controller\TemplateController'
31+
defaults:
32+
template: 'chat.html.twig'
33+
context: { chat: 'video' }
34+
2835
wikipedia:
2936
path: '/wikipedia'
3037
controller: 'Symfony\Bundle\FrameworkBundle\Controller\TemplateController'

‎demo.png

-37.4 KB
Loading

‎src/Video/TwigComponent.php

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace App\Video;
6+
7+
use PhpLlm\LlmChain\Bridge\OpenAI\GPT;
8+
use PhpLlm\LlmChain\Model\Message\Content\Image;
9+
use PhpLlm\LlmChain\Model\Message\Message;
10+
use PhpLlm\LlmChain\Model\Message\MessageBag;
11+
use PhpLlm\LlmChain\Model\Response\AsyncResponse;
12+
use PhpLlm\LlmChain\Model\Response\TextResponse;
13+
use PhpLlm\LlmChain\PlatformInterface;
14+
use Symfony\UX\LiveComponent\Attribute\AsLiveComponent;
15+
use Symfony\UX\LiveComponent\Attribute\LiveAction;
16+
use Symfony\UX\LiveComponent\Attribute\LiveArg;
17+
use Symfony\UX\LiveComponent\Attribute\LiveProp;
18+
use Symfony\UX\LiveComponent\DefaultActionTrait;
19+
20+
#[AsLiveComponent('video')]
21+
final class TwigComponent
22+
{
23+
use DefaultActionTrait;
24+
25+
#[LiveProp]
26+
public string $caption = 'Please define an instruction and hit submit.';
27+
28+
public function __construct(
29+
private readonly PlatformInterface $platform,
30+
) {
31+
}
32+
33+
#[LiveAction]
34+
public function submit(#[LiveArg] string $instruction, #[LiveArg] string $image): void
35+
{
36+
$messageBag = new MessageBag(
37+
Message::forSystem(<<<PROMPT
38+
You are a video captioning assistant. You are provided with a video frame and an instruction.
39+
You must generate a caption or answer based on the provided video frame and the user's instruction.
40+
You are not in a conversation with the user and there will be no follow-up questions or messages.
41+
PROMPT),
42+
Message::ofUser($instruction, Image::fromDataUrl($image))
43+
);
44+
45+
$response = $this->platform->request(new GPT(GPT::GPT_4O_MINI), $messageBag, [
46+
'max_tokens' => 100,
47+
]);
48+
49+
assert($response instanceof AsyncResponse);
50+
$response = $response->unwrap();
51+
assert($response instanceof TextResponse);
52+
53+
$this->caption = $response->getContent();
54+
}
55+
}

‎templates/base.html.twig

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@
3434
<li class="nav-item">
3535
<a class="nav-link" href="{{ path('audio') }}">{{ ux_icon('iconoir:microphone-solid', { height: '20px', width: '20px' }) }} Audio Bot</a>
3636
</li>
37+
<li class="nav-item">
38+
<a class="nav-link" href="{{ path('video') }}">{{ ux_icon('tabler:video-filled', { height: '20px', width: '20px' }) }} Video Bot</a>
39+
</li>
3740
<li class="nav-item"><span class="nav-link">|</span></li>
3841
<li class="nav-item">
3942
<a class="nav-link" href="https://github.com/php-llm/symfony-demo" target="_blank">{{ ux_icon('mdi:github', { height: '20px', width: '20px' }) }} GitHub</a>

‎templates/components/video.html.twig

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{% import "_message.html.twig" as message %}
2+
3+
<div class="card mx-auto shadow-lg" {{ attributes.defaults(stimulus_controller('video')) }}>
4+
<div class="card-header p-2">
5+
{{ ux_icon('tabler:video-filled', { height: '32px', width: '32px' }) }}
6+
<strong class="ms-1 pt-1 d-inline-block">Video Bot</strong>
7+
</div>
8+
<div id="chat-body" class="card-body p-2 overflow-auto">
9+
<div id="welcome" class="text-center mt-5 p-4 bg-white rounded-5 shadow-sm w-75 mx-auto">
10+
<div class="mb-2">
11+
<video id="videoFeed" autoplay playsinline></video>
12+
</div>
13+
<canvas id="canvas" class="d-none"></canvas>
14+
<i class="text-muted">{{ this.caption }}</i>
15+
</div>
16+
</div>
17+
<div class="card-footer p-2">
18+
<div class="input-group">
19+
<input id="chat-message" type="text" class="form-control border-0" placeholder="What do you see?">
20+
<button id="chat-submit" class="btn btn-outline-secondary border-0" type="button">{{ ux_icon('mingcute:send-fill', { height: '25px', width: '25px' }) }} Submit</button>
21+
</div>
22+
</div>
23+
</div>

‎templates/index.html.twig

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
</p>
1414
<h3 class="text-dark">Examples</h3>
1515
<div class="row">
16-
<div class="col-md-3">
16+
<div class="col-md-4">
1717
<div class="card blog bg-body shadow-sm">
1818
<div class="card-img-top py-2">
1919
{{ ux_icon('mdi:symfony', { height: '150px', width: '150px' }) }}
@@ -32,7 +32,7 @@
3232
{% endif %}
3333
</div>
3434
</div>
35-
<div class="col-md-3">
35+
<div class="col-md-4">
3636
<div class="card youtube bg-body shadow-sm">
3737
<div class="card-img-top py-2">
3838
{{ ux_icon('bi:youtube', { height: '150px', width: '150px' }) }}
@@ -51,7 +51,7 @@
5151
{% endif %}
5252
</div>
5353
</div>
54-
<div class="col-md-3">
54+
<div class="col-md-4">
5555
<div class="card wikipedia bg-body shadow-sm">
5656
<div class="card-img-top py-2">
5757
{{ ux_icon('mdi:wikipedia', { height: '150px', width: '150px' }) }}
@@ -70,14 +70,17 @@
7070
{% endif %}
7171
</div>
7272
</div>
73-
<div class="col-md-3">
73+
</div>
74+
<div class="row mt-4">
75+
<div class="col-md-2"></div>
76+
<div class="col-md-4">
7477
<div class="card audio bg-body shadow-sm">
7578
<div class="card-img-top py-2">
7679
{{ ux_icon('iconoir:microphone-solid', { height: '150px', width: '150px' }) }}
7780
</div>
7881
<div class="card-body">
7982
<h5 class="card-title">Audio Bot</h5>
80-
<p class="card-text">Simple demonstration of text-to-speech with Whisper in combination with GPT.</p>
83+
<p class="card-text">Simple demonstration of speech-to-text with Whisper in combination with GPT.</p>
8184
<a href="{{ path('audio') }}" class="btn btn-outline-dark d-block">Try Audio Bot</a>
8285
</div>
8386
{# Profiler route only available in dev #}
@@ -89,6 +92,26 @@
8992
{% endif %}
9093
</div>
9194
</div>
95+
<div class="col-md-4">
96+
<div class="card video bg-body shadow-sm">
97+
<div class="card-img-top py-2">
98+
{{ ux_icon('tabler:video-filled', { height: '150px', width: '150px' }) }}
99+
</div>
100+
<div class="card-body">
101+
<h5 class="card-title">Video Bot</h5>
102+
<p class="card-text">Simple demonstration of vision capabilities of GPT in combination with your webcam.</p>
103+
<a href="{{ path('video') }}" class="btn btn-outline-dark d-block">Try Video Bot</a>
104+
</div>
105+
{# Profiler route only available in dev #}
106+
{% if 'dev' == app.environment %}
107+
<div class="card-footer">
108+
{{ ux_icon('solar:code-linear', { height: '20px', width: '20px' }) }}
109+
<a href="{{ path('_profiler_open_file', { file: 'src/Video/TwigComponent.php', line: 37 }) }}">See Implementation</a>
110+
</div>
111+
{% endif %}
112+
</div>
113+
</div>
114+
<div class="col-md-2"></div>
92115
</div>
93116
</div>
94117
{% endblock %}

0 commit comments

Comments
 (0)