diff --git a/PluginBuilder/Controllers/HomeController.cs b/PluginBuilder/Controllers/HomeController.cs index ea2c422c..8a9b93a7 100644 --- a/PluginBuilder/Controllers/HomeController.cs +++ b/PluginBuilder/Controllers/HomeController.cs @@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.RateLimiting; +using Microsoft.Extensions.Diagnostics.HealthChecks; using Newtonsoft.Json.Linq; using PluginBuilder.APIModels; using PluginBuilder.Components.PluginVersion; @@ -33,7 +34,8 @@ public class HomeController( PluginBuilderOptions options, ServerEnvironment env, NostrService nostrService, - ILogger logger) + ILogger logger, + HealthCheckService healthCheckService) : Controller { [AllowAnonymous] @@ -793,4 +795,36 @@ public IActionResult Error() { return View("Error", new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier }); } + + [AllowAnonymous] + [HttpGet("/health")] + [EnableRateLimiting(Policies.PublicApiRateLimit)] + public async Task CheckHealth(CancellationToken cancellationToken) + { + var report = await healthCheckService.CheckHealthAsync(cancellationToken); + + var result = new + { + status = report.Status == HealthStatus.Healthy ? "UP" : "DOWN", + timestamp = DateTime.UtcNow, + description = report.Entries.Values.FirstOrDefault(e => e.Description is not null).Description + }; + + // display page + var acceptHeader = Request.Headers.Accept.ToString(); + if (acceptHeader.Contains("text/html")) + { + var hcvm = new HealthCheckViewModel { Healthy = result.status, Description = result.description ?? "" }; + + var view = View("HealthPage", hcvm); + view.StatusCode = report.Status == HealthStatus.Unhealthy + ? StatusCodes.Status503ServiceUnavailable + : StatusCodes.Status200OK; + + return view; + } + + // send JSON result + return report.Status == HealthStatus.Healthy ? Ok(result) : StatusCode(StatusCodes.Status503ServiceUnavailable, result); + } } diff --git a/PluginBuilder/HostedServices/AzureStartupHostedService.cs b/PluginBuilder/HostedServices/AzureStartupHostedService.cs index da74cf0a..a02000d1 100644 --- a/PluginBuilder/HostedServices/AzureStartupHostedService.cs +++ b/PluginBuilder/HostedServices/AzureStartupHostedService.cs @@ -4,6 +4,9 @@ namespace PluginBuilder.HostedServices; public class AzureStartupHostedService : IHostedService { + public static bool AzureStartupCompleted { get; private set; } + public static Exception? AzureStartupError { get; private set; } + public AzureStartupHostedService(AzureStorageClient azureStorageClient) { AzureStorageClient = azureStorageClient; @@ -13,7 +16,19 @@ public AzureStartupHostedService(AzureStorageClient azureStorageClient) public async Task StartAsync(CancellationToken cancellationToken) { - await AzureStorageClient.EnsureDefaultContainerExists(cancellationToken); + AzureStartupCompleted = false; + AzureStartupError = null; + + try + { + await AzureStorageClient.EnsureDefaultContainerExists(cancellationToken); + AzureStartupCompleted = true; + } + catch (Exception ex) + { + AzureStartupError = ex; + throw; + } } public Task StopAsync(CancellationToken cancellationToken) diff --git a/PluginBuilder/HostedServices/DatabaseStartupHostedService.cs b/PluginBuilder/HostedServices/DatabaseStartupHostedService.cs index 7b5973c9..48cc509d 100644 --- a/PluginBuilder/HostedServices/DatabaseStartupHostedService.cs +++ b/PluginBuilder/HostedServices/DatabaseStartupHostedService.cs @@ -13,6 +13,9 @@ public class DatabaseStartupHostedService : IHostedService { private readonly AdminSettingsCache _adminSettingsCache; + public static bool DatabaseStartupCompleted { get; private set; } + public static Exception? DatabaseStartupError { get; private set; } + public DatabaseStartupHostedService(ILogger logger, DBConnectionFactory connectionFactory, AdminSettingsCache adminSettingsCache) { @@ -27,6 +30,9 @@ public DatabaseStartupHostedService(ILogger logger public async Task StartAsync(CancellationToken cancellationToken) { + DatabaseStartupCompleted = false; + DatabaseStartupError = null; + retry: try { @@ -36,6 +42,8 @@ public async Task StartAsync(CancellationToken cancellationToken) await conn.SettingsInitialize(); await _adminSettingsCache.RefreshAllAdminSettings(conn); + + DatabaseStartupCompleted = true; } catch (NpgsqlException pgex) when (pgex.SqlState == "3D000") { @@ -52,6 +60,11 @@ public async Task StartAsync(CancellationToken cancellationToken) goto retry; } + catch (Exception ex) + { + DatabaseStartupError = ex; + throw; + } } public Task StopAsync(CancellationToken cancellationToken) diff --git a/PluginBuilder/HostedServices/DockerStartupHostedService.cs b/PluginBuilder/HostedServices/DockerStartupHostedService.cs index c8b3c543..88f3a38c 100644 --- a/PluginBuilder/HostedServices/DockerStartupHostedService.cs +++ b/PluginBuilder/HostedServices/DockerStartupHostedService.cs @@ -5,6 +5,26 @@ public class DockerStartupException : Exception public DockerStartupException(string message) : base(message) { } + + public static bool DockerStartupCompleted { get; private set; } + public static Exception? DockerStartupError { get; private set; } + + public static void ResetStartupState() + { + DockerStartupCompleted = false; + DockerStartupError = null; + } + + public static void MarkStartupCompleted() + { + DockerStartupCompleted = true; + } + + public static void MarkStartupFailed(Exception ex) + { + DockerStartupError = ex; + DockerStartupCompleted = false; + } } public class DockerStartupHostedService : IHostedService @@ -24,6 +44,8 @@ public DockerStartupHostedService(ILogger logger, IW public async Task StartAsync(CancellationToken cancellationToken) { + DockerStartupException.ResetStartupState(); + var skipBuildValue = Environment.GetEnvironmentVariable(SkipBuildEnvVar); var skipBuild = string.Equals(skipBuildValue, "1", StringComparison.OrdinalIgnoreCase) || string.Equals(skipBuildValue, "true", StringComparison.OrdinalIgnoreCase); @@ -31,26 +53,36 @@ public async Task StartAsync(CancellationToken cancellationToken) if (skipBuild) { Logger.LogInformation("Skipping docker image build because {SkipBuildEnvVar}=true", SkipBuildEnvVar); + DockerStartupException.MarkStartupCompleted(); return; } - Logger.LogInformation("Building the PluginBuilder docker image"); - - var buildResult = await ProcessRunner.RunAsync(new ProcessSpec + try { - Executable = "docker", - EnvironmentVariables = + Logger.LogInformation("Building the PluginBuilder docker image"); + + var buildResult = await ProcessRunner.RunAsync(new ProcessSpec { - // Somehow we get permission problem when buildkit isn't used - ["DOCKER_BUILDKIT"] = "1" - }, - Arguments = new[] { "build", "-f", "PluginBuilder.Dockerfile", "-t", "plugin-builder", "." }, - WorkingDirectory = ContentRootPath - }, cancellationToken); - if (buildResult != 0) - throw new DockerStartupException("The build of PluginBuilder.Dockerfile failed"); - - await CleanupDanglingBuildVolumes(cancellationToken); + Executable = "docker", + EnvironmentVariables = + { + // Somehow we get permission problem when buildkit isn't used + ["DOCKER_BUILDKIT"] = "1" + }, + Arguments = new[] { "build", "-f", "PluginBuilder.Dockerfile", "-t", "plugin-builder", "." }, + WorkingDirectory = ContentRootPath + }, cancellationToken); + if (buildResult != 0) + throw new DockerStartupException("The build of PluginBuilder.Dockerfile failed"); + + await CleanupDanglingBuildVolumes(cancellationToken); + DockerStartupException.MarkStartupCompleted(); + } + catch (Exception ex) + { + DockerStartupException.MarkStartupFailed(ex); + throw; + } } public Task StopAsync(CancellationToken cancellationToken) diff --git a/PluginBuilder/PluginBuilder.csproj b/PluginBuilder/PluginBuilder.csproj index 96a6c51b..e01f2455 100644 --- a/PluginBuilder/PluginBuilder.csproj +++ b/PluginBuilder/PluginBuilder.csproj @@ -123,6 +123,7 @@ + diff --git a/PluginBuilder/Program.cs b/PluginBuilder/Program.cs index f86e713a..1d580fcd 100644 --- a/PluginBuilder/Program.cs +++ b/PluginBuilder/Program.cs @@ -81,6 +81,8 @@ private void ConfigureBuilder(WebApplicationBuilder builder) if (!verbose) builder.Logging.AddFilter("Events", LogLevel.Warning); + builder.Services.AddHealthChecks().AddCheck("Dependencies"); + // Uncomment this to see EF queries // builder.Logging.AddFilter("Microsoft.EntityFrameworkCore.Database.Command", LogLevel.Trace); builder.Logging.AddFilter("Microsoft.EntityFrameworkCore.Migrations", LogLevel.Information); @@ -192,6 +194,7 @@ public void AddServices(IConfiguration configuration, IServiceCollection service services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddHttpClient(HttpClientNames.GitHub, client => { client.BaseAddress = new Uri("https://api.github.com/"); diff --git a/PluginBuilder/Services/AzureStorageClient.cs b/PluginBuilder/Services/AzureStorageClient.cs index 1119ac34..cfbcc590 100644 --- a/PluginBuilder/Services/AzureStorageClient.cs +++ b/PluginBuilder/Services/AzureStorageClient.cs @@ -62,6 +62,22 @@ public async Task EnsureDefaultContainerExists(CancellationToken cancellat return ToJson(output)["created"]!.Value(); } + public async Task IsDefaultContainerAccessible(CancellationToken cancellationToken = default) + { + try + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromSeconds(15)); + + var container = blobClient.GetContainerReference(DefaultContainer); + return await container.ExistsAsync(null, null, cts.Token); + } + catch + { + return false; + } + } + public async Task UploadImageFile(IFormFile file, string blobName) { var container = blobClient.GetContainerReference(DefaultContainer); diff --git a/PluginBuilder/Services/HealthService.cs b/PluginBuilder/Services/HealthService.cs new file mode 100644 index 00000000..87cdabdc --- /dev/null +++ b/PluginBuilder/Services/HealthService.cs @@ -0,0 +1,74 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Npgsql; + +namespace PluginBuilder.Services; + +public class HealthService : IHealthCheck +{ + public HealthService(DBConnectionFactory dbConnectionFactory, AzureStorageClient azureStorageClient, ProcessRunner processRunner, IHostApplicationLifetime lifetime) + { + DbConnectionFactory = dbConnectionFactory; + AzureStorageClient = azureStorageClient; + ProcessRunner = processRunner; + + _lifetime = lifetime; + } + + private readonly IHostApplicationLifetime _lifetime; + + private DBConnectionFactory DbConnectionFactory { get; } + private AzureStorageClient AzureStorageClient { get; } + private ProcessRunner ProcessRunner { get; } + + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + if (!_lifetime.ApplicationStarted.IsCancellationRequested) + return HealthCheckResult.Unhealthy("Startup incomplete"); + + var dbTask = IsDatabaseHealthy(cancellationToken); + var dockerTask = IsDockerHealthy(cancellationToken); + var azureTask = AzureStorageClient.IsDefaultContainerAccessible(cancellationToken); + + await Task.WhenAll(dbTask, dockerTask, azureTask); + + var isHealthy = dbTask.Result && dockerTask.Result && azureTask.Result; + + return isHealthy + ? HealthCheckResult.Healthy() + : HealthCheckResult.Unhealthy("Critical dependency unavailable"); + } + + private async Task IsDatabaseHealthy(CancellationToken cancellationToken) + { + try + { + await using var conn = await DbConnectionFactory.Open(cancellationToken); + await using var cmd = new NpgsqlCommand("SELECT 1", conn); + var result = await cmd.ExecuteScalarAsync(cancellationToken); + return result is 1; + } + catch + { + return false; + } + } + + private async Task IsDockerHealthy(CancellationToken cancellationToken) + { + try + { + var code = await ProcessRunner.RunAsync(new ProcessSpec + { + Executable = "docker", + Arguments = ["info", "--format", "{{ .ServerVersion }}"], + OutputCapture = new OutputCapture(), + ErrorCapture = new OutputCapture() + }, cancellationToken); + return code == 0; + } + catch + { + return false; + } + } +} diff --git a/PluginBuilder/ViewModels/Home/HealthCheckViewModel.cs b/PluginBuilder/ViewModels/Home/HealthCheckViewModel.cs new file mode 100644 index 00000000..3237207b --- /dev/null +++ b/PluginBuilder/ViewModels/Home/HealthCheckViewModel.cs @@ -0,0 +1,8 @@ +namespace PluginBuilder.ViewModels.Home; + +public class HealthCheckViewModel +{ + public string Healthy { get; set; } = string.Empty; + + public string Description { get; set; } = string.Empty; +} diff --git a/PluginBuilder/Views/Home/HealthPage.cshtml b/PluginBuilder/Views/Home/HealthPage.cshtml new file mode 100644 index 00000000..777099bd --- /dev/null +++ b/PluginBuilder/Views/Home/HealthPage.cshtml @@ -0,0 +1,65 @@ +@model HealthCheckViewModel + +@{ + Layout = "_LayoutPublicModal"; + ViewData["Title"] = "Health"; + var isHealthy = Model.Healthy == "UP"; +} + + + +
+
+

+ @if (isHealthy) + { + All Systems Operational! + } + else + { + Service Disruption + } +

+ +
+ + @if (isHealthy) + { +

Uncle says it's all right.

+ +
+ + Server UP + +


+ +

+ @if (User.Identity?.IsAuthenticated == true) + { + Head back to the public plugin page. + } + else + { + Nothing to see here. You can log in if you dare. + } +

+ } + else + { +

+ Something went wrong. +

+ +
+ + @if (!string.IsNullOrWhiteSpace(@Model.Description)) + { +

Error: @Model.Description

+ } + +

+ +

Please try again later or contact support here if the issue persists.

+ } +
+
diff --git a/PluginBuilder/wwwroot/img/healthpage/up-successkid.jpg b/PluginBuilder/wwwroot/img/healthpage/up-successkid.jpg new file mode 100644 index 00000000..bcb2c69d Binary files /dev/null and b/PluginBuilder/wwwroot/img/healthpage/up-successkid.jpg differ diff --git a/PluginBuilder/wwwroot/img/healthpage/up-uncleok.jpg b/PluginBuilder/wwwroot/img/healthpage/up-uncleok.jpg new file mode 100644 index 00000000..e092f60d Binary files /dev/null and b/PluginBuilder/wwwroot/img/healthpage/up-uncleok.jpg differ