Skip to content
This repository was archived by the owner on Nov 27, 2024. It is now read-only.
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit aba18da

Browse files
saddam213saddam213
authoredJan 31, 2024
Merge pull request #108 from saddam213/VideoExample
VideoToVideo example
2 parents 3de7398 + 8832f49 commit aba18da

File tree

7 files changed

+102
-21
lines changed

7 files changed

+102
-21
lines changed
 

‎Assets/Samples/Input.gif

4.9 MB
Loading

‎Assets/Samples/Output_VideoToVideo.gif

5.88 MB
Loading

‎OnnxStack.Console/Examples/VideoToVideoExample.cs

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,9 @@ public VideoToVideoExample(StableDiffusionConfig configuration, IVideoService vi
2929

3030
public async Task RunAsync()
3131
{
32-
string inputVideoPath = "C:\\Users\\Deven\\Pictures\\gidsgphy.gif";
33-
var inputFile = File.ReadAllBytes(inputVideoPath);
34-
var videoInfo = await _videoService.GetVideoInfoAsync(inputFile);
35-
var videoInput = await _videoService.CreateFramesAsync(inputFile, videoInfo.FPS);
32+
// Load Video
33+
var targetFPS = 15;
34+
var videoInput = await VideoInput.FromFileAsync("C:\\Users\\Deven\\Pictures\\gidsgphy.gif", targetFPS);
3635

3736
// Loop though the appsettings.json model sets
3837
foreach (var modelSet in _configuration.ModelSets)
@@ -50,18 +49,15 @@ public async Task RunAsync()
5049
{
5150
Prompt = "Iron Man",
5251
DiffuserType = DiffuserType.ImageToImage,
53-
InputVideo = new VideoInput(videoInput)
52+
InputVideo = videoInput
5453
};
5554

5655
// Run pipeline
5756
var result = await pipeline.RunAsync(promptOptions, progressCallback: OutputHelpers.FrameProgressCallback);
5857

59-
// Create Video from Tensor result
60-
var videoResult = await _videoService.CreateVideoAsync(result, videoInfo.FPS);
61-
6258
// Save Video File
6359
var outputFilename = Path.Combine(_outputDirectory, $"{modelSet.Name}.mp4");
64-
await File.WriteAllBytesAsync(outputFilename, videoResult.Data);
60+
await VideoInput.SaveFileAsync(result, outputFilename, targetFPS);
6561
}
6662
}
6763
}

‎OnnxStack.Core/Services/IVideoService.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public interface IVideoService
4545
/// <param name="videoFPS">The video FPS.</param>
4646
/// <param name="cancellationToken">The cancellation token.</param>
4747
/// <returns></returns>
48-
Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float videoFPS, CancellationToken cancellationToken = default);
48+
Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float? videoFPS = default, CancellationToken cancellationToken = default);
4949

5050

5151
/// <summary>
@@ -55,7 +55,7 @@ public interface IVideoService
5555
/// <param name="videoFPS">The video FPS.</param>
5656
/// <param name="cancellationToken">The cancellation token.</param>
5757
/// <returns></returns>
58-
Task<VideoFrames> CreateFramesAsync(Stream videoStream, float videoFPS, CancellationToken cancellationToken = default);
58+
Task<VideoFrames> CreateFramesAsync(Stream videoStream, float? videoFPS = default, CancellationToken cancellationToken = default);
5959

6060

6161
/// <summary>
@@ -67,7 +67,7 @@ public interface IVideoService
6767
/// <returns></returns>
6868
/// <exception cref="NotSupportedException">VideoTensor not supported</exception>
6969
/// <exception cref="ArgumentException">No video data found</exception>
70-
Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float videoFPS, CancellationToken cancellationToken = default);
70+
Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float? videoFPS = default, CancellationToken cancellationToken = default);
7171

7272

7373
/// <summary>

‎OnnxStack.Core/Services/VideoService.cs

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ public async Task<VideoOutput> CreateVideoAsync(IEnumerable<byte[]> videoFrames,
135135
/// <returns></returns>
136136
/// <exception cref="NotSupportedException">VideoTensor not supported</exception>
137137
/// <exception cref="ArgumentException">No video data found</exception>
138-
public async Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float videoFPS, CancellationToken cancellationToken = default)
138+
public async Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float? videoFPS = default, CancellationToken cancellationToken = default)
139139
{
140140

141141
if (videoInput.VideoBytes is not null)
@@ -158,11 +158,12 @@ public async Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float vi
158158
/// <param name="videoFPS">The video FPS.</param>
159159
/// <param name="cancellationToken">The cancellation token.</param>
160160
/// <returns></returns>
161-
public async Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float videoFPS, CancellationToken cancellationToken = default)
161+
public async Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float? videoFPS = default, CancellationToken cancellationToken = default)
162162
{
163163
var videoInfo = await GetVideoInfoAsync(videoBytes, cancellationToken);
164-
var videoFrames = await CreateFramesInternalAsync(videoBytes, videoFPS, cancellationToken).ToListAsync(cancellationToken);
165-
videoInfo = videoInfo with { FPS = videoFPS };
164+
var targetFPS = videoFPS ?? videoInfo.FPS;
165+
var videoFrames = await CreateFramesInternalAsync(videoBytes, targetFPS, cancellationToken).ToListAsync(cancellationToken);
166+
videoInfo = videoInfo with { FPS = targetFPS };
166167
return new VideoFrames(videoInfo, videoFrames);
167168
}
168169

@@ -174,14 +175,16 @@ public async Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float videoF
174175
/// <param name="videoFPS">The video FPS.</param>
175176
/// <param name="cancellationToken">The cancellation token.</param>
176177
/// <returns></returns>
177-
public async Task<VideoFrames> CreateFramesAsync(Stream videoStream, float videoFPS, CancellationToken cancellationToken = default)
178+
public async Task<VideoFrames> CreateFramesAsync(Stream videoStream, float? videoFPS = default, CancellationToken cancellationToken = default)
178179
{
179180
using (var memoryStream = new MemoryStream())
180181
{
181182
await memoryStream.CopyToAsync(videoStream, cancellationToken).ConfigureAwait(false);
182183
var videoBytes = memoryStream.ToArray();
183184
var videoInfo = await GetVideoInfoAsync(videoBytes, cancellationToken);
184-
var videoFrames = await CreateFramesInternalAsync(videoBytes, videoFPS, cancellationToken).ToListAsync(cancellationToken);
185+
var targetFPS = videoFPS ?? videoInfo.FPS;
186+
var videoFrames = await CreateFramesInternalAsync(videoBytes, targetFPS, cancellationToken).ToListAsync(cancellationToken);
187+
videoInfo = videoInfo with { FPS = targetFPS };
185188
return new VideoFrames(videoInfo, videoFrames);
186189
}
187190
}

‎OnnxStack.Core/Video/VideoInput.cs

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
1-
using Microsoft.ML.OnnxRuntime.Tensors;
1+
using Microsoft.Extensions.Primitives;
2+
using Microsoft.ML.OnnxRuntime.Tensors;
3+
using OnnxStack.Core.Config;
4+
using OnnxStack.Core.Services;
25
using System.IO;
36
using System.Text.Json.Serialization;
7+
using System.Threading;
8+
using System.Threading.Tasks;
49

510
namespace OnnxStack.Core.Video
611
{
@@ -34,7 +39,7 @@ public VideoInput() { }
3439
/// </summary>
3540
/// <param name="videoFrames">The video frames.</param>
3641
public VideoInput(VideoFrames videoFrames) => VideoFrames = videoFrames;
37-
42+
3843

3944
/// <summary>
4045
/// Gets the video bytes.
@@ -75,5 +80,39 @@ public VideoInput() { }
7580
|| VideoStream != null
7681
|| VideoTensor != null
7782
|| VideoFrames != null;
83+
84+
85+
86+
/// <summary>
87+
/// Create a VideoInput from file
88+
/// </summary>
89+
/// <param name="videoFile">The video file.</param>
90+
/// <param name="targetFPS">The target FPS.</param>
91+
/// <param name="config">The configuration.</param>
92+
/// <param name="cancellationToken">The cancellation token.</param>
93+
/// <returns></returns>
94+
public static async Task<VideoInput> FromFileAsync(string videoFile, float? targetFPS = default, OnnxStackConfig config = default, CancellationToken cancellationToken = default)
95+
{
96+
var videoBytes = await File.ReadAllBytesAsync(videoFile, cancellationToken);
97+
var videoService = new VideoService(config ?? new OnnxStackConfig());
98+
var videoFrames = await videoService.CreateFramesAsync(videoBytes, targetFPS, cancellationToken);
99+
return new VideoInput(videoFrames);
100+
}
101+
102+
103+
/// <summary>
104+
/// Saves the video file
105+
/// </summary>
106+
/// <param name="videoTensor">The video tensor.</param>
107+
/// <param name="videoFile">The video file.</param>
108+
/// <param name="targetFPS">The target FPS.</param>
109+
/// <param name="config">The configuration.</param>
110+
/// <param name="cancellationToken">The cancellation token.</param>
111+
public static async Task SaveFileAsync(DenseTensor<float> videoTensor, string videoFile, float targetFPS, OnnxStackConfig config = default, CancellationToken cancellationToken = default)
112+
{
113+
var videoService = new VideoService(config ?? new OnnxStackConfig());
114+
var videoOutput = await videoService.CreateVideoAsync(videoTensor, targetFPS, cancellationToken);
115+
await File.WriteAllBytesAsync(videoFile, videoOutput.Data, cancellationToken);
116+
}
78117
}
79118
}

‎OnnxStack.StableDiffusion/README.md

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,4 +175,47 @@ await pipeline.UnloadAsync();
175175
```
176176
| Input | Output |
177177
| :--- | :--- |
178-
<img src="../Assets/Samples/Input_Depth.png" width="256"/> | <img src="../Assets/Samples/Output_ControlNet.png" width="256"/>
178+
<img src="../Assets/Samples/Input_Depth.png" width="256"/> | <img src="../Assets/Samples/Output_ControlNet.png" width="256"/>
179+
180+
181+
182+
183+
## Stable Diffusion VideoToVideo Example
184+
Run Stable Diffusion process on a video frame by frame
185+
```csharp
186+
//Model:
187+
//https://huggingface.co/runwayml/stable-diffusion-v1-5 (onnx branch)
188+
189+
// Create Pipeline
190+
var pipeline = StableDiffusionPipeline.CreatePipeline("models\\stable-diffusion-v1-5");
191+
192+
// Preload Models (optional)
193+
await pipeline.LoadAsync();
194+
195+
// Load Video
196+
var targetFPS = 15;
197+
var videoInput = await VideoInput.FromFileAsync("Input.gif", targetFPS);
198+
199+
// Add text and video to prompt
200+
var promptOptions = new PromptOptions
201+
{
202+
Prompt = "Elon Musk",
203+
DiffuserType = DiffuserType.ImageToImage,
204+
InputVideo = videoInput
205+
};
206+
207+
// Run pipeline
208+
var result = await pipeline.RunAsync(promptOptions, progressCallback: OutputHelpers.FrameProgressCallback);
209+
210+
// Save Video File
211+
var outputFilename = Path.Combine(_outputDirectory, "Output_VideoToVideo.mp4");
212+
await VideoInput.SaveFileAsync(result, outputFilename, targetFPS);
213+
214+
// Unload Pipleine
215+
await pipeline.UnloadAsync();
216+
```
217+
218+
| Input | Output |
219+
| :--- | :--- |
220+
<img src="../Assets/Samples/Input.gif" width="256"/> | <img src="../Assets/Samples/Output_VideoToVideo.gif" width="256"/>
221+
_converted to gif for github readme_ | _converted to gif for github readme_

0 commit comments

Comments
 (0)
This repository has been archived.