diff --git a/.gitignore b/.gitignore index 0f27a213..6c256e99 100644 --- a/.gitignore +++ b/.gitignore @@ -361,3 +361,5 @@ AppsMonitoring/**/appsettings.Secret.json AppsMonitoring/**/data.db AppsMonitoring/infra/appsettings.*.json AppsMonitoring/infra/**/result.yaml + +AppsAnalysis/config.ini diff --git a/AppsAnalysis/.csharpierignore b/AppsAnalysis/.csharpierignore new file mode 100644 index 00000000..7bd4dc47 --- /dev/null +++ b/AppsAnalysis/.csharpierignore @@ -0,0 +1 @@ +**/repos/ \ No newline at end of file diff --git a/AppsAnalysis/.editorconfig b/AppsAnalysis/.editorconfig new file mode 100644 index 00000000..e31fe572 --- /dev/null +++ b/AppsAnalysis/.editorconfig @@ -0,0 +1,34 @@ +[*.cs] + +# IDE0058: Expression value is never used +dotnet_diagnostic.IDE0058.severity = suggestion + +# S125: Sections of code should not be commented out +dotnet_diagnostic.S125.severity = suggestion + +# S112: General exceptions should never be thrown +dotnet_diagnostic.S112.severity = suggestion + +# CA2201: Exception type is not sufficiently specific +dotnet_diagnostic.CA2201.severity = suggestion + +# S1135: Track uses of "TODO" tags +dotnet_diagnostic.S1135.severity = suggestion + +# S4487: Unread private class field +dotnet_diagnostic.S4487.severity = suggestion + +# S1481: Unused local variables should be removed +dotnet_diagnostic.S1481.severity = suggestion + +# CA1050: Declare types in namespaces +dotnet_diagnostic.CA1050.severity = suggestion + +# S3903: Types should be defined in named namespaces +dotnet_diagnostic.S3903.severity = suggestion + +# S108: Nested blocks of code should not be left empty +dotnet_diagnostic.S108.severity = suggestion + +# S1199: Nested code blocks should not be used +dotnet_diagnostic.S1199.severity = suggestion diff --git a/AppsAnalysis/AppsAnalysis.sln b/AppsAnalysis/AppsAnalysis.sln new file mode 100644 index 00000000..6dfd1843 --- /dev/null +++ b/AppsAnalysis/AppsAnalysis.sln @@ -0,0 +1,34 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{C17303EC-5744-4990-806D-9FAAD4040DE9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Altinn.Analysis", "src\Altinn.Analysis\Altinn.Analysis.csproj", "{108D9742-E8FE-4DA4-B8B9-E6CC849DD3ED}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Altinn.Analysis.Cli", "src\Altinn.Analysis.Cli\Altinn.Analysis.Cli.csproj", "{54883DAB-F56B-4005-871E-4BADA984405C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {108D9742-E8FE-4DA4-B8B9-E6CC849DD3ED}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {108D9742-E8FE-4DA4-B8B9-E6CC849DD3ED}.Debug|Any CPU.Build.0 = Debug|Any CPU + {108D9742-E8FE-4DA4-B8B9-E6CC849DD3ED}.Release|Any CPU.ActiveCfg = Release|Any CPU + {108D9742-E8FE-4DA4-B8B9-E6CC849DD3ED}.Release|Any CPU.Build.0 = Release|Any CPU + {54883DAB-F56B-4005-871E-4BADA984405C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {54883DAB-F56B-4005-871E-4BADA984405C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {54883DAB-F56B-4005-871E-4BADA984405C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {54883DAB-F56B-4005-871E-4BADA984405C}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {108D9742-E8FE-4DA4-B8B9-E6CC849DD3ED} = {C17303EC-5744-4990-806D-9FAAD4040DE9} + {54883DAB-F56B-4005-871E-4BADA984405C} = {C17303EC-5744-4990-806D-9FAAD4040DE9} + EndGlobalSection +EndGlobal diff --git a/AppsAnalysis/Directory.Build.props b/AppsAnalysis/Directory.Build.props new file mode 100644 index 00000000..61f63877 --- /dev/null +++ b/AppsAnalysis/Directory.Build.props @@ -0,0 +1,32 @@ + + + enable + enable + latest + true + Recommended + strict + true + true + false + NU1901;NU1902;NU1903;NU1904 + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + + all + runtime; build; native; contentfiles; analyzers + + + all + runtime; build; native; contentfiles; analyzers + + + diff --git a/AppsAnalysis/README.md b/AppsAnalysis/README.md new file mode 100644 index 00000000..25e336d6 --- /dev/null +++ b/AppsAnalysis/README.md @@ -0,0 +1,45 @@ +# Apps analysis + +Meant to gather statistics on code from all apps in Altinn 3. + +## Prereqs + +A config file at `config.ini` +```sh +cp config.sample.ini config.ini +``` +Make the appropriate changes. +The password needs to be a token from Altinn Studio with `Read` permissions for `organization` and `repository` scope. +See below: +![Access token config](img/access-token-config.png) + +Example +```ini +username=martinothamar +password=8e************************ +directory=C:\code\digdir\altinn\analysis +clear_directory=true +``` + +This would target Altinn Studio prod by default. + +## Run + +Go into CLI directory: +```sh +cd src/Altinn.Analysis.Cli +``` + +Fetch +```sh +dotnet run -c Release -- fetch -c ../../config.ini +``` + +![Downloading apps](img/download-progress.png) + +Analyze +```sh +dotnet run -c Release -- analyze -c ../../config.ini +``` + +![Analyzing apps](img/analysis.png) diff --git a/AppsAnalysis/config.sample.ini b/AppsAnalysis/config.sample.ini new file mode 100644 index 00000000..a4e49f6e --- /dev/null +++ b/AppsAnalysis/config.sample.ini @@ -0,0 +1,6 @@ +directory= +altinn_url= +username= +password= +clear_directory= +max_parallelism= diff --git a/AppsAnalysis/img/access-token-config.png b/AppsAnalysis/img/access-token-config.png new file mode 100644 index 00000000..c46b4513 Binary files /dev/null and b/AppsAnalysis/img/access-token-config.png differ diff --git a/AppsAnalysis/img/analysis.png b/AppsAnalysis/img/analysis.png new file mode 100644 index 00000000..af9061d1 Binary files /dev/null and b/AppsAnalysis/img/analysis.png differ diff --git a/AppsAnalysis/img/download-progress.png b/AppsAnalysis/img/download-progress.png new file mode 100644 index 00000000..38cf22c7 Binary files /dev/null and b/AppsAnalysis/img/download-progress.png differ diff --git a/AppsAnalysis/src/Altinn.Analysis.Cli/Altinn.Analysis.Cli.csproj b/AppsAnalysis/src/Altinn.Analysis.Cli/Altinn.Analysis.Cli.csproj new file mode 100644 index 00000000..30204006 --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis.Cli/Altinn.Analysis.Cli.csproj @@ -0,0 +1,25 @@ + + + + Exe + net8.0 + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + + + + + diff --git a/AppsAnalysis/src/Altinn.Analysis.Cli/Program.cs b/AppsAnalysis/src/Altinn.Analysis.Cli/Program.cs new file mode 100644 index 00000000..1111e4f9 --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis.Cli/Program.cs @@ -0,0 +1,158 @@ +using System.Diagnostics; +using System.Globalization; +using Altinn.Analysis; +using ConsoleAppFramework; +using Spectre.Console; + +var app = ConsoleApp.Create(); +app.Add(); +await app.RunAsync(args); + +public class Commands +{ +#pragma warning disable CA1822 // Mark members as static + + /// Fetch Altinn apps into a local directory + /// -u, username for Altinn Studio. + /// -p, password for Altinn Studio. + /// -d, local directory to place fetched apps. + /// -cd, clear directory before fetching. + /// -mp, make parallelism for running analysis (including fetching data). + /// -au, URL for Altinn Studio APIs. + /// -c, Config file for this CLI. + [Command("fetch")] + public async Task Fetch( + string? username = null, + string? password = null, + string? directory = null, + bool? clearDirectory = null, + int? maxParallelism = null, + string? altinnUrl = null, + string? configFile = null, + CancellationToken cancellationToken = default + ) + { + var timer = Stopwatch.StartNew(); + var config = await TryReadConfigFile(configFile, cancellationToken); + using var analyzer = new AppsFetcher( + new( + directory ?? config.Directory ?? "repos/", + username ?? config.Username ?? throw new Exception("Need username config to fetch"), + password ?? config.Password ?? throw new Exception("Need password config to fetch"), + maxParallelism ?? config.MaxParallelism ?? Environment.ProcessorCount, + clearDirectory ?? config.ClearDirectory ?? false, + altinnUrl ?? config.AltinnUrl ?? "https://altinn.studio" + ) + ); + await analyzer.Fetch(cancellationToken); + timer.Stop(); + var elapsed = timer.Elapsed switch + { + var v when v >= TimeSpan.FromHours(1) => $"{v.TotalHours:0.00}h", + var v when v >= TimeSpan.FromMinutes(1) => $"{v.TotalMinutes:0.00}m", + var v => $"{v.TotalSeconds:0.00}s", + }; + AnsiConsole.MarkupLine(""); + AnsiConsole.MarkupLine($"[green]Fetch completed in {elapsed}[/]"); + } + + /// Analyze Altinn apps in a directory + /// -d, local directory to place fetched apps. + /// -mp, make parallelism for running analysis (including fetching data). + /// -c, Config file for this CLI. + [Command("analyze")] + public async Task Analyze( + string? directory = null, + int? maxParallelism = null, + string? configFile = null, + CancellationToken cancellationToken = default + ) + { + var timer = Stopwatch.StartNew(); + var config = await TryReadConfigFile(configFile, cancellationToken); + var analyzer = new AppsAnalyzer( + new( + directory ?? config.Directory ?? "repos/", + maxParallelism ?? config.MaxParallelism ?? Environment.ProcessorCount + ) + ); + await analyzer.Analyze(cancellationToken); + timer.Stop(); + var elapsed = timer.Elapsed switch + { + var v when v >= TimeSpan.FromHours(1) => $"{v.TotalHours:0.00}h", + var v when v >= TimeSpan.FromMinutes(1) => $"{v.TotalMinutes:0.00}m", + var v => $"{v.TotalSeconds:0.00}s", + }; + AnsiConsole.MarkupLine(""); + AnsiConsole.MarkupLine($"[green]Analysis completed in {elapsed}[/]"); + } + + private static async Task TryReadConfigFile( + string? configFile, + CancellationToken cancellationToken + ) + { + var result = new ConfigFile(); + + if (string.IsNullOrWhiteSpace(configFile)) + return result; + + if (!File.Exists(configFile)) + throw new Exception($"Could not find config file at: '{configFile}'"); + + var lines = await File.ReadAllLinesAsync(configFile, cancellationToken); + foreach (var line in lines) + { + if (string.IsNullOrWhiteSpace(line)) + continue; + + if (line.Split('=') is not [var key, var value]) + throw new Exception($"Invalid config at line: {line}"); + + switch (key) + { + case "directory": + result = result with { Directory = value }; + break; + case "username": + result = result with { Username = value }; + break; + case "password": + result = result with { Password = value }; + break; + case "clear_directory": + if (!bool.TryParse(value, out var clearDirectory)) + throw new Exception( + $"Unexpected value for 'clear_directory': '{value}' - expected boolean" + ); + result = result with { ClearDirectory = clearDirectory }; + break; + case "max_parallelism": + if (!int.TryParse(value, CultureInfo.InvariantCulture, out var maxParallelism)) + throw new Exception( + $"Unexpected value for 'max_parallelism': '{value}' - expected integer" + ); + result = result with { MaxParallelism = maxParallelism }; + break; + case "altinn_url": + result = result with { AltinnUrl = value }; + break; + default: + throw new Exception($"Unexpected key in config: '{key}'"); + } + } + + return result; + } + + private readonly record struct ConfigFile + { + public string? Directory { get; init; } + public string? Username { get; init; } + public string? Password { get; init; } + public bool? ClearDirectory { get; init; } + public int? MaxParallelism { get; init; } + public string? AltinnUrl { get; init; } + } +} diff --git a/AppsAnalysis/src/Altinn.Analysis.Cli/Properties/launchSettings.json b/AppsAnalysis/src/Altinn.Analysis.Cli/Properties/launchSettings.json new file mode 100644 index 00000000..b04b06e0 --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis.Cli/Properties/launchSettings.json @@ -0,0 +1,18 @@ +{ + "profiles": { + "Fetch": { + "commandName": "Project", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "commandLineArgs": "fetch -c $(MSBuildProjectDirectory)/../../config.ini" + }, + "Analyze": { + "commandName": "Project", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "commandLineArgs": "analyze -c $(MSBuildProjectDirectory)/../../config.ini" + }, + } +} \ No newline at end of file diff --git a/AppsAnalysis/src/Altinn.Analysis/Altinn.Analysis.csproj b/AppsAnalysis/src/Altinn.Analysis/Altinn.Analysis.csproj new file mode 100644 index 00000000..d528c637 --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis/Altinn.Analysis.csproj @@ -0,0 +1,17 @@ + + + + net8.0 + + + + + + + + + + + + + diff --git a/AppsAnalysis/src/Altinn.Analysis/AppAnalyzer.cs b/AppsAnalysis/src/Altinn.Analysis/AppAnalyzer.cs new file mode 100644 index 00000000..6f473974 --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis/AppAnalyzer.cs @@ -0,0 +1,291 @@ +using System.Collections.Concurrent; +using System.Collections.Specialized; +using Buildalyzer; +using Buildalyzer.Workspaces; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.FindSymbols; + +namespace Altinn.Analysis; + +public sealed record AppRepository(DirectoryInfo Dir, string Org, string Name); + +// TODO: Consider SoA if this approach works OK +public readonly record struct AppAnalysisResult +{ + private static readonly BitVector32.Section _timedOut = BitVector32.CreateSection(2); + private static readonly BitVector32.Section _validProject = BitVector32.CreateSection( + 2, + _timedOut + ); + private static readonly BitVector32.Section _builds = BitVector32.CreateSection( + 2, + _validProject + ); + private static readonly BitVector32.Section _hasAppLib = BitVector32.CreateSection(2, _builds); + private static readonly BitVector32.Section _hasLatestAppLib = BitVector32.CreateSection( + 2, + _hasAppLib + ); + + private readonly BitVector32 _bits; + + public AppAnalysisResult( + AppRepository repo, + bool? timedOut = null, + bool? validProject = null, + bool? builds = null, + bool? hasAppLib = null, + bool? HasLatestAppLib = null, + string? appLibVersion = null, + uint? warningCount = null, + Dictionary>? symbolReferenceCounts = null + ) + { + AppRepository = repo; + _bits = new BitVector32(0); + _bits[_timedOut] = ToValue(timedOut); + _bits[_validProject] = ToValue(validProject); + _bits[_builds] = ToValue(builds); + _bits[_hasAppLib] = ToValue(hasAppLib); + _bits[_hasLatestAppLib] = ToValue(HasLatestAppLib); + AppLibVersion = appLibVersion; + WarningCount = warningCount; + SymbolReferenceCountsBySymbol = symbolReferenceCounts ?? new(); + } + + public AppRepository AppRepository { get; } + + public readonly IReadOnlyDictionary< + string, + List + > SymbolReferenceCountsBySymbol { get; } + + public string? AppLibVersion { get; } + + public uint? WarningCount { get; } + + public readonly bool? TimedOut => FromValue(_timedOut); + public readonly bool? ValidProject => FromValue(_validProject); + public readonly bool? Builds => FromValue(_builds); + public readonly bool? HasAppLib => FromValue(_hasAppLib); + public readonly bool? HasLatestAppLib => FromValue(_hasLatestAppLib); + + public bool OK => + (_bits[_validProject] & _bits[_builds] & _bits[_hasAppLib] & _bits[_hasLatestAppLib]) == 1; + + private static int ToValue(bool? value) => + value switch + { + null => 2, + false => 0, + true => 1, + }; + + private bool? FromValue(BitVector32.Section section) => + _bits[section] switch + { + 2 => default(bool?), + 0 => false, + 1 => true, + var u => throw new Exception($"Unexpected value: {u}"), + }; +} + +public sealed class AppAnalyzer : IDisposable +{ + private readonly SyncThreadPool _threadPool; + + public AppAnalyzer(int threadCount) + { + _threadPool = new SyncThreadPool(threadCount); + } + + public async Task Analyze( + AppRepository app, + string[] findReferences, + ConcurrentBag results, + CancellationToken cancellationToken + ) + { + var result = await _threadPool.RunAsync( + async (CancellationToken cancellationToken) => + { + var manager = new AnalyzerManager(); + var dir = app.Dir; + var projectFile = Path.Combine(dir.FullName, "App", "App.csproj"); + + if (!File.Exists(projectFile)) + return new AppAnalysisResult(app, timedOut: false, validProject: false); + + IAnalyzerResults buildResults; + IProjectAnalyzer project; + try + { + project = manager.GetProject(projectFile); + cancellationToken.ThrowIfCancellationRequested(); + + buildResults = project.Build(); + cancellationToken.ThrowIfCancellationRequested(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return new AppAnalysisResult(app, timedOut: false, validProject: false); + } + + var builds = buildResults.OverallSuccess; + var result = buildResults.FirstOrDefault(r => r.Succeeded); + if (result is null) + return new AppAnalysisResult( + app, + timedOut: false, + validProject: true, + builds: builds + ); + + var appLibRef = result.PackageReferences.FirstOrDefault(r => + // Main + r.Key == "Altinn.App.Api" + // PR releases + || r.Key == "Altinn.App.Api.Experimental" + // <=6.0 + || r.Key == "Altinn.App.Common" + ); + string? version = null; + var hasAppLib = + appLibRef.Key is not null + && appLibRef.Value.TryGetValue("Version", out version); + var HasLatestAppLib = version is not null && version.StartsWith('8'); + if (!builds || !hasAppLib || !HasLatestAppLib) + return new AppAnalysisResult( + app, + timedOut: false, + validProject: true, + builds: builds, + hasAppLib: hasAppLib, + HasLatestAppLib: HasLatestAppLib, + appLibVersion: version + ); + + using var workspace = new AdhocWorkspace(); + // workspace.WorkspaceFailed += (sender, args) => logger?.LogError("Workspace failed: {Diagnostic}{NewLine}", args.Diagnostic, System.Environment.NewLine); + Project roslynProject; + Compilation? compilation; + try + { + result.AddToWorkspace(workspace, addProjectReferences: true); + roslynProject = workspace.CurrentSolution.Projects.First(); + compilation = await roslynProject.GetCompilationAsync(cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return new AppAnalysisResult( + app, + timedOut: false, + validProject: true, + builds: false, + hasAppLib: hasAppLib, + HasLatestAppLib: HasLatestAppLib, + appLibVersion: version + ); + } + + if (compilation is null) + return new AppAnalysisResult( + app, + timedOut: false, + validProject: true, + builds: false, + hasAppLib: hasAppLib, + HasLatestAppLib: HasLatestAppLib, + appLibVersion: version + ); + + var diagnostics = compilation.GetDiagnostics(cancellationToken); + var warningCount = (uint) + diagnostics.Count(d => + d.Id != "CS1701" && d.Severity == DiagnosticSeverity.Warning + ); + builds = + builds + && !diagnostics.Any(d => + d.Severity >= DiagnosticSeverity.Error && !d.IsSuppressed + ); + if (!builds) + return new AppAnalysisResult( + app, + timedOut: false, + validProject: true, + builds: builds, + hasAppLib: hasAppLib, + HasLatestAppLib: HasLatestAppLib, + appLibVersion: version, + warningCount: warningCount + ); + + var referenceCounts = new Dictionary>(16); + await PopulateReferenceCounts( + findReferences, + roslynProject, + compilation, + referenceCounts, + cancellationToken + ); + + return new AppAnalysisResult( + app, + timedOut: false, + validProject: true, + builds: builds, + hasAppLib: hasAppLib, + HasLatestAppLib: HasLatestAppLib, + appLibVersion: version, + warningCount: warningCount, + symbolReferenceCounts: referenceCounts + ); + }, + cancellationToken + ); + results.Add(result); + } + + private static async Task PopulateReferenceCounts( + string[] findReferences, + Project roslynProject, + Compilation compilation, + Dictionary> referenceCounts, + CancellationToken cancellationToken + ) + { + foreach (var symbolRef in findReferences) + { + if (!referenceCounts.TryGetValue(symbolRef, out var symbolRefs)) + referenceCounts[symbolRef] = symbolRefs = new List(16); + + var symbols = DocumentationCommentId.GetSymbolsForDeclarationId(symbolRef, compilation); + foreach (var symbol in symbols) + { + var references = await SymbolFinder.FindReferencesAsync( + symbol, + roslynProject.Solution, + cancellationToken + ); + + foreach (var reference in references) + { + // Filter out references to the symbol itself + // (sometimes we get both the interface method and the implementation method) + if (!SymbolEqualityComparer.Default.Equals(reference.Definition, symbol)) + continue; + + foreach (var location in reference.Locations) + symbolRefs.Add(reference); + } + } + } + } + + public void Dispose() + { + _threadPool.Dispose(); + } +} diff --git a/AppsAnalysis/src/Altinn.Analysis/AppsAnalyzer.cs b/AppsAnalysis/src/Altinn.Analysis/AppsAnalyzer.cs new file mode 100644 index 00000000..16139fdb --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis/AppsAnalyzer.cs @@ -0,0 +1,365 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using CsvHelper; +using Microsoft.CodeAnalysis.Text; +using Spectre.Console; + +namespace Altinn.Analysis; + +public sealed record AnalysisConfig(string Directory, int MaxParallelism); + +public sealed class AppsAnalyzer +{ + private readonly AnalysisConfig _config; + + private DirectoryInfo? _directory; + private int _parallelism; + + public AppsAnalyzer(AnalysisConfig config) + { + _config = config; + } + + private bool VerifyConfig() + { + var table = new Table(); + table.Border(TableBorder.None); + table.AddColumn(new TableColumn("")); + table.AddColumn(new TableColumn("")); + + _parallelism = _config.MaxParallelism; + var directory = Path.GetFullPath( + _config.Directory.Replace( + "~", + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) + ) + ); + table.AddRow( + new Markup(nameof(FetchConfig.Directory)), + new Markup($"= [bold]{directory}[/]") + ); + table.AddRow(new Markup("Parallelism"), new Markup($"= [bold]{_parallelism}[/]")); + AnsiConsole.Write(table); + AnsiConsole.WriteLine(); + + if (!Debugger.IsAttached) + { + var proceed = AnsiConsole.Prompt(new ConfirmationPrompt($"Continue?")); + return proceed; + } + + return true; + } + + public async Task Analyze(CancellationToken cancellationToken) + { + AnsiConsole.MarkupLine("[blue]Analyzing Altinn apps[/]... Config:"); + if (!VerifyConfig()) + return; + + var directory = Path.GetFullPath( + _config.Directory.Replace( + "~", + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) + ) + ); + _directory = new DirectoryInfo(directory); + if (!_directory.Exists) + { + AnsiConsole.MarkupLine( + $"Directory for downloaded apps [red]doesn't exist[/]: [bold]{directory}[/]" + ); + return; + } + + var repos = GetRepos(cancellationToken); + + AnsiConsole.WriteLine(); + AnsiConsole.Write(new Rule($"Analysing [blue]{repos.Count}[/] apps").LeftJustified()); + + AppAnalysisResult[]? results = null; + await AnsiConsole + .Progress() + .Columns( + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new ElapsedTimeColumn(), + new SpinnerColumn() + ) + .HideCompleted(true) + .AutoClear(true) + .StartAsync(async ctx => + { + results = await AnalyzeRepos(ctx, repos, cancellationToken); + }); + + Debug.Assert(results is not null); + AnsiConsole.MarkupLine($"[green]Successfully[/] analyzed [blue]{results.Length}[/] apps"); + + // Ample opportunity for SIMD below... + var timedOut = results.Count(r => r.TimedOut is true); + var invalidProjects = results.Count(r => r.ValidProject is false); + var failedBuilds = results.Count(r => r.Builds is false); + var noAppLib = results.Count(r => r.HasAppLib is false); + var oldAppLib = results.Count(r => r.HasLatestAppLib is false); + var aOkay = results.Count(r => r.OK); + + AnsiConsole.MarkupLine("[green]Overview[/]"); + AnsiConsole.WriteLine(); + AnsiConsole.Write( + new BarChart() + .Width(60) + .Label("[green bold underline]Analysis[/]") + .LeftAlignLabel() + .AddItem("Timed out", timedOut, Color.Red) + .AddItem("Invalid project", invalidProjects, Color.Red) + .AddItem("Failed builds", failedBuilds, Color.Red) + .AddItem("Missing applib", noAppLib, Color.Red) + .AddItem("Old applib", oldAppLib, Color.Yellow) + .AddItem("On applib v8", aOkay, Color.Green) + ); + + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[green]Symbol references totals[/]"); + var symbolReferences = results + .SelectMany(r => + r.SymbolReferenceCountsBySymbol.Select(kvp => + (Symbol: kvp.Key, References: kvp.Value.AsReadOnly(), App: r.AppRepository) + ) + ) + .GroupBy(r => r.Symbol) + .Select(grp => new + { + Symbol = grp.Key, + Count = grp.Sum(x => x.References.Count), + Apps = string.Join( + ", ", + grp.Where(x => x.References.Count > 0) + .Select(x => $"{x.App.Org}/{x.App.Name}") + .Order() + ), + }) + .OrderByDescending(x => x.Count) + .ToArray(); + + var table = new Table(); + table.Border(TableBorder.Rounded); + table.AddColumn(new TableColumn("Symbol")); + table.AddColumn(new TableColumn("Reference count")); + table.AddColumn(new TableColumn("Apps")); + + foreach (var item in symbolReferences) + { + table.AddRow(item.Symbol, item.Count.ToString(CultureInfo.InvariantCulture), item.Apps); + } + AnsiConsole.Write(table); + + { + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[blue]Writing code dump for references[/]..."); + var data = results + .SelectMany(re => + re.SymbolReferenceCountsBySymbol.SelectMany(kvp => + kvp.Value.Select(r => + (Symbol: kvp.Key, App: re.AppRepository, Reference: r) + ) + ) + ) + .GroupBy(r => r.Symbol); + + foreach (var item in data) + { + var refToFind = item.Key; + var filePath = Path.Join( + _directory.FullName, + $"dump_{refToFind.Substring(2).Replace('.', '_')}.md" + ); + if (File.Exists(filePath)) + File.Delete(filePath); + await using var file = File.OpenWrite(filePath); + await using var writer = new StreamWriter(file); + foreach (var reference in item) + { + foreach (var location in reference.Reference.Locations) + { + var documentPath = location.Document.FilePath; + var code = await location.Document.GetTextAsync(cancellationToken); + var lineNumber = code + .Lines.GetLineFromPosition(location.Location.SourceSpan.Start) + .LineNumber; + var codeLink = $"{documentPath} {lineNumber + 1}"; + var span = ExpandSpanToIncludeSurroundingLines( + code, + location.Location.SourceSpan + ); + var codeWithContext = code.GetSubText(span); + await writer.WriteLineAsync( + $"\n{codeLink}: \n```csharp\n{codeWithContext.ToString()}\n```\n" + ); + } + } + } + + AnsiConsole.MarkupLine($"[green]Successfully[/] wrote code dumps for references"); + } + + { + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[blue]Writing CSV report[/]..."); + + // Write CSV report to analysis directory + var filename = Path.Combine(_directory.FullName, "apps.csv"); + await using var writer = new StreamWriter(filename); + await using var csv = new CsvWriter(writer, CultureInfo.InvariantCulture); + + var records = results + .Select(r => new AppCsvRow + { + Org = r.AppRepository.Org, + Name = r.AppRepository.Name, + TimedOut = r.TimedOut, + Builds = r.Builds, + AppLibVersion = r.AppLibVersion, + WarningCount = r.WarningCount, + }) + .ToArray(); + await csv.WriteRecordsAsync(records, cancellationToken); + + AnsiConsole.MarkupLine( + $"[green]Successfully[/] wrote CSV report to: [bold]{filename}[/]" + ); + } + } + + private List GetRepos(CancellationToken cancellationToken) + { + Debug.Assert(_directory is not null); + + var result = new List(64); + var orgsCounter = 0; + foreach (var orgDir in Directory.EnumerateDirectories(_directory.FullName)) + { + var org = orgDir.Substring(orgDir.LastIndexOf(Path.DirectorySeparatorChar) + 1); + cancellationToken.ThrowIfCancellationRequested(); + + var reposCounter = 0; + foreach (var repoDir in Directory.EnumerateDirectories(orgDir)) + { + var name = repoDir.Substring(repoDir.LastIndexOf(Path.DirectorySeparatorChar) + 1); + var repoDirInfo = new DirectoryInfo( + Path.Combine(repoDir, Constants.MainBranchFolder) + ); + if (!repoDirInfo.Exists) + { + AnsiConsole.MarkupLine( + $"Couldn't find main branch folder for repo: '{repoDirInfo.Name}'" + ); + continue; + } + result.Add(new AppRepository(repoDirInfo, org, name)); + reposCounter++; + + if (reposCounter == Constants.LimitRepos) + break; + } + + orgsCounter++; + + if (orgsCounter == Constants.LimitOrgs) + break; + } + + return result; + } + + private async Task AnalyzeRepos( + ProgressContext ctx, + List repos, + CancellationToken cancellationToken + ) + { + using var appAnalyzer = new AppAnalyzer(_parallelism); + + var results = new ConcurrentBag(); + var options = new ParallelOptions + { + CancellationToken = cancellationToken, + MaxDegreeOfParallelism = _parallelism, + }; + await Parallel.ForEachAsync( + repos, + options, + async (repo, cancellationToken) => + { + var (dir, org, name) = repo; + var task = ctx.AddTask($"[green]{org}/{name}[/]", autoStart: false, maxValue: 1); + task.IsIndeterminate = true; + task.StartTask(); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(TimeSpan.FromMinutes(1)); + cancellationToken = cts.Token; + + // Doc for selecting refs: + // https://github.com/dotnet/roslyn/blob/7ead97fc404947f81a4ee3c6543212d60fddfd03/src/RoslynAnalyzers/Microsoft.CodeAnalysis.BannedApiAnalyzers/BannedApiAnalyzers.Help.md + string[] findRefs = + [ + // "M:Altinn.App.Core.Internal.Auth.IAuthorizationClient.GetUserRoles(System.Int32,System.Int32)", + // "T:Altinn.App.Core.Features.IValidator", + "T:Altinn.App.Core.Infrastructure.Clients.Storage.DataClient", + "M:Altinn.App.Core.Infrastructure.Clients.Storage.DataClient.#ctor", + // "T:Altinn.App.Core.Features.IProcessTaskEnd", + ]; + try + { + await appAnalyzer.Analyze(repo, findRefs, results, cancellationToken); + } + catch (OperationCanceledException) + { + results.Add(new(repo, timedOut: true)); + } + + task.Increment(1.0); + task.StopTask(); + } + ); + + return results.ToArray(); + } + + private sealed record AppCsvRow + { + public required string Org { get; init; } + public required string Name { get; init; } + public required bool? TimedOut { get; init; } + public required bool? Builds { get; init; } + public required string? AppLibVersion { get; init; } + public required uint? WarningCount { get; init; } + } + + private static TextSpan ExpandSpanToIncludeSurroundingLines( + SourceText sourceText, + TextSpan originalSpan + ) + { + // Get the lines that contain the start and end of the original span + var startLine = sourceText.Lines.GetLineFromPosition(originalSpan.Start); + var endLine = sourceText.Lines.GetLineFromPosition(originalSpan.End); + + // Get the line numbers + int startLineNumber = startLine.LineNumber; + int endLineNumber = endLine.LineNumber; + + // Expand to include one line before and after (with bounds checking) + int expandedStartLineNumber = Math.Max(0, startLineNumber - 1); + int expandedEndLineNumber = Math.Min(sourceText.Lines.Count - 1, endLineNumber + 1); + + // Get the expanded lines + var expandedStartLine = sourceText.Lines[expandedStartLineNumber]; + var expandedEndLine = sourceText.Lines[expandedEndLineNumber]; + + // Create the expanded TextSpan + return TextSpan.FromBounds(expandedStartLine.Start, expandedEndLine.End); + } +} diff --git a/AppsAnalysis/src/Altinn.Analysis/AppsFetcher.cs b/AppsAnalysis/src/Altinn.Analysis/AppsFetcher.cs new file mode 100644 index 00000000..06908662 --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis/AppsFetcher.cs @@ -0,0 +1,401 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using System.Threading.Channels; +using Spectre.Console; + +namespace Altinn.Analysis; + +public sealed record FetchConfig( + string Directory, + string Username, + string Password, + int MaxParallelism, + bool ClearDirectory, + string AltinnUrl +); + +public sealed class AppsFetcher : IDisposable +{ + private readonly FetchConfig _config; + private readonly GiteaClient _giteaClient; + private readonly KubernetesWrapperClient _kubernetesWrapperClient; + + private DirectoryInfo? _directory; + private int _parallelism; + + public AppsFetcher(FetchConfig config) + { + _config = config; + _giteaClient = new GiteaClient(config); + _kubernetesWrapperClient = new KubernetesWrapperClient(); + } + + private bool VerifyConfig() + { + var table = new Table(); + table.Border(TableBorder.None); + table.AddColumn(new TableColumn("")); + table.AddColumn(new TableColumn("")); + + _parallelism = Math.Min(Constants.LimitMaxParallelism, _config.MaxParallelism); + var clearDirectory = _config.ClearDirectory; + var clearDirectoryConf = clearDirectory + .ToString(CultureInfo.InvariantCulture) + .ToLowerInvariant(); + var directory = Path.GetFullPath( + _config.Directory.Replace( + "~", + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) + ) + ); + table.AddRow( + new Markup(nameof(FetchConfig.Directory)), + new Markup($"= [bold]{directory}[/]") + ); + table.AddRow( + new Markup(nameof(FetchConfig.Username)), + new Markup($"= [bold]{_config.Username}[/]") + ); + table.AddRow( + new Markup(nameof(FetchConfig.Password)), + new Markup($"= [bold]{_config.Password[..2]}**********[/]") + ); + table.AddRow( + new Markup(nameof(FetchConfig.ClearDirectory)), + new Markup($"= [bold]{clearDirectoryConf}[/]") + ); + table.AddRow(new Markup("Parallelism"), new Markup($"= [bold]{_parallelism}[/]")); + table.AddRow( + new Markup(nameof(FetchConfig.AltinnUrl)), + new Markup($"= [bold]{_config.AltinnUrl}[/]") + ); + AnsiConsole.Write(table); + AnsiConsole.WriteLine(); + + if (!Debugger.IsAttached) + { + var proceed = AnsiConsole.Prompt(new ConfirmationPrompt($"Continue?")); + return proceed; + } + + return true; + } + + private void PrepareDirectory() + { + var directory = Path.GetFullPath( + _config.Directory.Replace( + "~", + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) + ) + ); + _directory = new DirectoryInfo(directory); + + if (_config.ClearDirectory && _directory.Exists) + { + AnsiConsole.MarkupLine($"Clearing directory '{_directory.FullName}'..."); + _directory.Delete(recursive: true); + } + + if (!_directory.Exists) + _directory = Directory.CreateDirectory(directory); + } + + public async Task Fetch(CancellationToken cancellationToken) + { + AnsiConsole.MarkupLine("[blue]Fetching Altinn apps[/]... Config:"); + if (!VerifyConfig()) + return; + + PrepareDirectory(); + + var apps = await GetApps(cancellationToken); + + AnsiConsole.WriteLine(); + if (!Debugger.IsAttached) + { + var proceed = AnsiConsole.Prompt( + new ConfirmationPrompt($"Continue to download apps locally?") + ); + if (!proceed) + return; + } + + await DownloadApps(apps, cancellationToken); + } + + private sealed record OrgRecord(List Repos) + { + public int Skipped { get; set; } + } + + private async Task GetApps(CancellationToken cancellationToken) + { + AnsiConsole.WriteLine(); + AnsiConsole.Write(new Rule("Finding organizations and apps").LeftJustified()); + GetAppsResult? result = null; + await AnsiConsole + .Progress() + .Columns( + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new ValueProgressColumn(), + new ElapsedTimeColumn(), + new SpinnerColumn() + ) + .HideCompleted(true) + .AutoClear(true) + .StartAsync(async ctx => + { + result = await GetApps(ctx, cancellationToken); + }); + // Debug.Assert(result is not null); + // ! TODO + var (orgs, repos, reposByOrg) = result!; + var skippedTotal = reposByOrg.Sum(r => r.Value.Skipped); + AnsiConsole.MarkupLine( + $"Fetched organizations and apps from Altinn Studio - [green]{orgs.Count} organizations[/], [blue]{repos.Count} apps[/] (skipped {skippedTotal} undeployed apps)" + ); + return result; + } + + private async Task GetApps( + ProgressContext ctx, + CancellationToken cancellationToken + ) + { + AnsiConsole.MarkupLine($"Fetching organizations and apps from Altinn Studio Gitea"); + + ConcurrentDictionary reposByOrg = new(); + + var options = new ParallelOptions + { + CancellationToken = cancellationToken, + MaxDegreeOfParallelism = _parallelism, + }; + await Parallel.ForEachAsync( + _giteaClient.GetOrgs(cancellationToken), + options, + async (org, canncellationToken) => + { + var task = ctx.AddTask($"[green]{org.FullName} ({org.Name})[/]", autoStart: false); + + task.IsIndeterminate = true; + task.StartTask(); + + var deploymentstt02 = await _kubernetesWrapperClient.GetDeployments( + org.Name, + "tt02" + ); + var deploymentsProd = await _kubernetesWrapperClient.GetDeployments( + org.Name, + "prod" + ); + HashSet deployedApps = new(deploymentstt02.Select(d => d.Repo)); + deployedApps.UnionWith(deploymentsProd.Select(d => d.Repo)); + + task.MaxValue = deployedApps.Count; + + var orgRecord = reposByOrg.GetOrAdd(org, _ => new(new List(4))); + + await foreach (var repo in _giteaClient.GetRepos(org.Name, cancellationToken)) + { + if (deployedApps.Contains(repo.Name)) + orgRecord.Repos.Add(repo); + else + orgRecord.Skipped += 1; + var maxValue = task.MaxValue; + if (orgRecord.Repos.Count >= maxValue) + task.MaxValue = maxValue + 25; + task.Increment(1); + } + task.MaxValue = orgRecord.Repos.Count; + + ctx.Refresh(); + task.StopTask(); + } + ); + + return new GetAppsResult( + reposByOrg.Keys.ToArray(), + reposByOrg.SelectMany(kvp => kvp.Value.Repos).ToArray(), + reposByOrg.ToDictionary( + kvp => kvp.Key, + kvp => ((IReadOnlyList)kvp.Value.Repos, kvp.Value.Skipped) + ) + ); + } + + private async Task DownloadApps( + GetAppsResult apps, + CancellationToken cancellationToken + ) + { + var (orgs, repos, _) = apps; + AnsiConsole.WriteLine(); + AnsiConsole.Write(new Rule($"Downloading apps").LeftJustified()); + + DownloadAppsResult? result = null; + await AnsiConsole + .Progress() + .Columns( + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new ValueProgressColumn(), + new PercentageColumn(), + new ElapsedTimeColumn(), + new SpinnerColumn() + ) + .HideCompleted(true) + .AutoClear(true) + .StartAsync(async ctx => + { + result = await DownloadApps(ctx, apps, cancellationToken); + }); + + Debug.Assert(result is not null); + + AnsiConsole.MarkupLine( + $"[green]Successfully[/] fetched [blue]{result.Repos.Count}[/] apps" + ); + + return result; + } + + private async Task DownloadApps( + ProgressContext ctx, + GetAppsResult basicInformation, + CancellationToken cancellationToken + ) + { + Debug.Assert(_directory is not null); + + var queue = Channel.CreateBounded<( + GiteaOrg Org, + GiteaRepo Repo, + ProgressTask Task, + DirectoryInfo OrgDirectory + )>( + new BoundedChannelOptions(_parallelism * 2) + { + SingleWriter = true, + SingleReader = false, + } + ); + + var results = new ConcurrentBag(); + + var processors = new Task[_parallelism]; + for (int i = 0; i < _parallelism; i++) + { + processors[i] = Task.Run( + async () => + { + var reader = queue.Reader; + while (await reader.WaitToReadAsync(cancellationToken)) + { + while (reader.TryRead(out var entry)) + { + var (org, repo, task, orgDirectory) = entry; + var branch = repo.DefaultBranch; + if (string.IsNullOrWhiteSpace(branch)) + throw new Exception( + $"Default branch not set for repo: '{repo.FullName}'" + ); + + var repoDirectory = orgDirectory.CreateSubdirectory(repo.Name); + var branchDirectory = repoDirectory.CreateSubdirectory( + Constants.MainBranchFolder + ); + + if ( + branchDirectory.Exists + && new DirectoryInfo( + Path.Join(branchDirectory.FullName, ".git") + ).Exists + ) + { + await _giteaClient.UpdateRepo( + repo, + branchDirectory, + cancellationToken + ); + } + else + { + await _giteaClient.CloneRepo( + repo, + branchDirectory, + branch, + cancellationToken: cancellationToken + ); + } + + results.Add(new(org, repo, branch, branchDirectory)); + + task.Increment(1); + + if (task.IsFinished) + task.StopTask(); + } + } + }, + cancellationToken + ); + } + + var options = new ParallelOptions + { + CancellationToken = cancellationToken, + MaxDegreeOfParallelism = _parallelism, + }; + await Parallel.ForEachAsync( + basicInformation.Orgs, + options, + async (org, cancellationToken) => + { + var (repos, skipped) = basicInformation.ReposByOrg[org]; + var task = ctx.AddTask( + $"[green]{org.FullName} ({org.Name})[/] (skipped: {skipped})", + autoStart: true, + maxValue: repos.Count + ); + var orgDirectory = _directory.CreateSubdirectory(org.Name); + foreach (var repo in repos) + { + await queue.Writer.WriteAsync( + (org, repo, task, orgDirectory), + cancellationToken + ); + } + } + ); + + queue.Writer.Complete(); + + await Task.WhenAll(processors); + + return new DownloadAppsResult(results.ToArray()); + } + + public void Dispose() + { + _giteaClient.Dispose(); + } + + private sealed record GetAppsResult( + IReadOnlyList Orgs, + IReadOnlyList Repos, + IReadOnlyDictionary Repos, int Skipped)> ReposByOrg + ); + + private sealed record DownloadAppsResult(IReadOnlyList Repos); + + private sealed record RepoInfo( + GiteaOrg Org, + GiteaRepo Repo, + string Branch, + DirectoryInfo MainDir + ); +} diff --git a/AppsAnalysis/src/Altinn.Analysis/Constants.cs b/AppsAnalysis/src/Altinn.Analysis/Constants.cs new file mode 100644 index 00000000..21b67d21 --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis/Constants.cs @@ -0,0 +1,11 @@ +namespace Altinn.Analysis; + +internal static class Constants +{ + internal const string MainBranchFolder = "main"; + + // Following settings are to tweak during test-runs + internal static readonly int LimitOrgs = -1; + internal static readonly int LimitRepos = -1; + internal static readonly int LimitMaxParallelism = 8; +} diff --git a/AppsAnalysis/src/Altinn.Analysis/GiteaClient.cs b/AppsAnalysis/src/Altinn.Analysis/GiteaClient.cs new file mode 100644 index 00000000..0e59665f --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis/GiteaClient.cs @@ -0,0 +1,352 @@ +using System.Globalization; +using System.IO.Compression; +using System.Net; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Web; +using LibGit2Sharp; +using Spectre.Console; + +namespace Altinn.Analysis; + +public sealed record GiteaOrg( + [property: JsonPropertyName("id")] long Id, + [property: JsonPropertyName("avatar_url")] string? AvatarUrl, + [property: JsonPropertyName("name")] string Name, + [property: JsonPropertyName("full_name")] string? FullName, + [property: JsonPropertyName("location")] string? Location, + [property: JsonPropertyName("description")] string? Description, + [property: JsonPropertyName("email")] string? Email, + [property: JsonPropertyName("visibility")] string? Visibility +) +{ + public bool Equals(GiteaOrg? other) => Id == other?.Id; + + public override int GetHashCode() => Id.GetHashCode(); +} + +public sealed record GiteaRepo( + [property: JsonPropertyName("id")] long Id, + [property: JsonPropertyName("clone_url")] string CloneUrl, + [property: JsonPropertyName("ssh_url")] string? SshUrl, + [property: JsonPropertyName("url")] string? Url, + [property: JsonPropertyName("name")] string Name, + [property: JsonPropertyName("full_name")] string FullName, + [property: JsonPropertyName("default_branch")] string? DefaultBranch, + [property: JsonPropertyName("link")] string? Link, + [property: JsonPropertyName("private")] bool? Private +) +{ + private string[]? _split; + + [JsonIgnore] + public string Org + { + get + { + _split ??= FullName.Split('/'); + return _split[0]; + } + } + + public bool Equals(GiteaOrg? other) => Id == other?.Id; + + public override int GetHashCode() => Id.GetHashCode(); +} + +internal sealed record GiteaArchiveStatus([property: JsonPropertyName("complete")] bool Complete); + +[JsonSourceGenerationOptions] +[JsonSerializable(typeof(GiteaOrg))] +[JsonSerializable(typeof(GiteaRepo))] +[JsonSerializable(typeof(GiteaArchiveStatus))] +internal sealed partial class GiteaClientJsonContext : JsonSerializerContext { } + +public sealed class GiteaClient : IDisposable +{ + private const int PageSize = 50; + private const string PageSizeStr = "50"; + + private readonly HttpClient _httpClient; + private readonly SyncThreadPool _threadPool; + + private readonly FetchConfig _config; + + public GiteaClient(FetchConfig config) + { + _config = config; + HttpClient httpClient = new( + new SocketsHttpHandler + { + PooledConnectionLifetime = TimeSpan.FromMinutes(15), // Recreate every 15 minutes + } + ); + httpClient.BaseAddress = new Uri(config.AltinnUrl); + httpClient.DefaultRequestHeaders.Add("Accept", "application/json"); + httpClient.DefaultRequestHeaders.Add("Authorization", $"token {config.Password}"); + _httpClient = httpClient; + _threadPool = new SyncThreadPool(config.MaxParallelism); + } + + public async IAsyncEnumerable GetOrgs( + [EnumeratorCancellation] CancellationToken cancellationToken = default + ) + { + var page = 1; + while (true) + { + var pageParam = page.ToString(CultureInfo.InvariantCulture); + var url = $"/repos/api/v1/orgs?page={pageParam}&limit={PageSizeStr}"; + + using var request = new HttpRequestMessage(HttpMethod.Get, url); + using var response = await _httpClient.SendAsync( + request, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken + ); + + response.EnsureSuccessStatusCode(); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + + var enumerable = JsonSerializer.DeserializeAsyncEnumerable( + stream, + GiteaClientJsonContext.Default.GiteaOrg, + cancellationToken: cancellationToken + ); + + var counter = 0; + await foreach (var org in enumerable) + { + if (org is null) + throw new JsonException("Got 'null' org in response"); + if (string.IsNullOrWhiteSpace(org.Name)) + throw new JsonException($"Got invalid name for org with ID: {org.Id}"); + + yield return org; + counter++; + + if (counter == Constants.LimitOrgs) + break; + } + + if (counter < PageSize) + { + break; + } + + page++; + } + } + + public async IAsyncEnumerable GetRepos( + string org, + [EnumeratorCancellation] CancellationToken cancellationToken = default + ) + { + var page = 1; + while (true) + { + var pageParam = page.ToString(CultureInfo.InvariantCulture); + var orgParam = HttpUtility.UrlEncode(org); + var url = $"/repos/api/v1/orgs/{orgParam}/repos?page={pageParam}&limit={PageSizeStr}"; + + using var request = new HttpRequestMessage(HttpMethod.Get, url); + using var response = await _httpClient.SendAsync( + request, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken + ); + + response.EnsureSuccessStatusCode(); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + + var enumerable = JsonSerializer.DeserializeAsyncEnumerable( + stream, + GiteaClientJsonContext.Default.GiteaRepo, + cancellationToken: cancellationToken + ); + + var counter = 0; + await foreach (var repo in enumerable) + { + if (repo is null) + throw new JsonException("Got 'null' repo in response"); + + yield return repo; + counter++; + + if (counter == Constants.LimitRepos) + break; + } + + if (counter < PageSize) + { + break; + } + + page++; + } + } + + public Task CloneRepo( + GiteaRepo repo, + DirectoryInfo path, + string? branch = null, + CancellationToken cancellationToken = default + ) + { + return _threadPool.RunAsync( + (CancellationToken cancellationToken) => + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + var options = new CloneOptions(); + options.FetchOptions.CredentialsProvider = (_url, _user, _cred) => + new UsernamePasswordCredentials + { + Username = _config.Username, + Password = _config.Password, + }; + options.FetchOptions.Depth = 1; + if (!string.IsNullOrWhiteSpace(branch)) + options.BranchName = branch; + + return Repository.Clone(repo.CloneUrl, path.FullName, options); + } + catch (Exception ex) + { + AnsiConsole.MarkupLine( + $"Error cloning repo '{repo.CloneUrl}' with branch '{branch}' to '{path.FullName}': {ex.Message}" + ); + return ""; + } + }, + cancellationToken + ); + } + + public async Task UpdateRepo( + GiteaRepo repo, + DirectoryInfo path, + CancellationToken cancellationToken = default + ) + { + await _threadPool.RunAsync( + (CancellationToken cancellationToken) => + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + using var repository = new Repository(path.FullName); + + // Fetch the latest with depth 1 + var fetchOptions = new FetchOptions + { + Depth = 1, + CredentialsProvider = (_url, _user, _cred) => + new UsernamePasswordCredentials + { + Username = _config.Username, + Password = _config.Password, + }, + }; + + Commands.Fetch( + repository, + "origin", + repository + .Network.Remotes["origin"] + .FetchRefSpecs.Select(x => x.Specification), + fetchOptions, + "Fetching latest" + ); + + // Hard reset to the remote branch + var remoteBranch = repository.Branches[ + $"origin/{repository.Head.FriendlyName}" + ]; + repository.Reset(ResetMode.Hard, remoteBranch.Tip); + + return true; + } + catch (Exception ex) + { + AnsiConsole.MarkupLine($"Error updating repo '{repo.FullName}': {ex.Message}"); + return false; + } + }, + cancellationToken + ); + } + + public async Task DownloadRepoArchive( + GiteaOrg org, + GiteaRepo repo, + DirectoryInfo path, + string branch, + CancellationToken cancellationToken = default + ) + { + var url = $"/repos/api/v1/repos/{org.Name}/{repo.Name}/archive/{branch}.zip"; + + using var request = new HttpRequestMessage(HttpMethod.Get, url); + using var response = await _httpClient.SendAsync( + request, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken + ); + + if (response.StatusCode == HttpStatusCode.NotFound) + { + // AnsiConsole.MarkupLine($"Couldn't download branch '{branch}' for repo '{repo.FullName}'"); + // Some repos are just completely empty + return; + } + + response.EnsureSuccessStatusCode(); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken); + + using var zipStream = new ZipArchive(stream, ZipArchiveMode.Read); + + foreach (var entry in zipStream.Entries) + { + if (entry.Name == "") + continue; + + await using var entryStream = entry.Open(); + + // Make sure we are consistent with directory separator chars + var entryFullPath = + Path.DirectorySeparatorChar == '\\' + ? entry.FullName.Replace('/', '\\') + : entry.FullName; + + // Archive entry FullName starts with repo name/dir, but we just want all the contents of the + // repo inside the input path, so we strip out the "root" directory from the full name + var firstDirSlash = entryFullPath.IndexOf(Path.DirectorySeparatorChar); + var entryPath = Path.Combine(path.FullName, entryFullPath.Substring(firstDirSlash + 1)); + + // Make sure the directory we want to write to exists + var directory = + Path.GetDirectoryName(entryPath) + ?? throw new Exception("Couldn't get archive entry directory"); + if (!Directory.Exists(directory)) + Directory.CreateDirectory(directory); + + await using var entryFileStream = File.Open( + entryPath, + FileMode.CreateNew, + FileAccess.Write, + FileShare.Read + ); + await entryStream.CopyToAsync(entryFileStream, cancellationToken); + } + } + + public void Dispose() + { + _threadPool.Dispose(); + } +} diff --git a/AppsAnalysis/src/Altinn.Analysis/KubernetesWrapperClient.cs b/AppsAnalysis/src/Altinn.Analysis/KubernetesWrapperClient.cs new file mode 100644 index 00000000..c5f467fc --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis/KubernetesWrapperClient.cs @@ -0,0 +1,84 @@ +using System.Net.Http.Json; +using System.Net.Sockets; +using System.Text.Json.Serialization; + +namespace Altinn.Analysis; + +// {"version":"1.2.15","release":"krt-krt-1001a-1" + +public sealed record Deployment(string Version, string Repo); + +public sealed class KubernetesWrapperClient +{ + private readonly HttpClient _httpClient; + + public KubernetesWrapperClient() + { + HttpClient httpClient = new( + new SocketsHttpHandler + { + PooledConnectionLifetime = TimeSpan.FromMinutes(15), // Recreate every 15 minutes + ConnectTimeout = TimeSpan.FromSeconds(5), + } + ); + httpClient.DefaultRequestHeaders.Add("Accept", "application/json"); + _httpClient = httpClient; + } + + private static string BaseUrl(string org, string env) => + $"https://{org}.apps{(env == "prod" ? "" : "." + env)}.altinn.no"; + + public async Task> GetDeployments(string org, string env) + { + var url = $"{BaseUrl(org, env)}/kuberneteswrapper/api/v1/deployments"; + + IReadOnlyList? response = null; + try + { + response = await _httpClient.GetFromJsonAsync>(url); + } + catch (HttpRequestException e) + when (e.InnerException is SocketException { SocketErrorCode: var errorCode } + && (errorCode is SocketError.HostNotFound or SocketError.TimedOut) + ) + { + // There is an org in CDN, but they have no DNS record, probably aren't any deployments... + return []; + } + catch (TaskCanceledException) + { + // Could be connect timeout + return []; + } + + if (response is null) + throw new Exception($"Could not get deployments for: {org}, {env}"); + + var deployments = new List(response.Count); + for (int i = 0; i < response.Count; i++) + { + var deployment = response[i]; + if (string.IsNullOrWhiteSpace(deployment.Release)) + continue; + + if (deployment.Release == "kuberneteswrapper") + continue; + + var split = deployment.Release.Split('-', 2, StringSplitOptions.RemoveEmptyEntries); + if (split.Length != 2) + throw new Exception( + $"Invalid deployment release string format for: {org}, {env} - {deployment.Release}" + ); + var appId = split[1]; + + deployments.Add(new Deployment(deployment.Version, appId)); + } + + return deployments; + } + + internal sealed record DeploymentInfoDto( + [property: JsonPropertyName("version")] string Version, + [property: JsonPropertyName("release")] string Release + ); +} diff --git a/AppsAnalysis/src/Altinn.Analysis/SyncThreadPool.cs b/AppsAnalysis/src/Altinn.Analysis/SyncThreadPool.cs new file mode 100644 index 00000000..125a669d --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis/SyncThreadPool.cs @@ -0,0 +1,119 @@ +using System.Threading.Channels; + +namespace Altinn.Analysis; + +/// +/// Threadpool for running synchronous work +/// Passes `TaskCreationOptions.LongRunning` to individual task/threads +/// to make sure the scheduler spawns additional threads for each task. +/// Scheduling sync work on default TaskScheduler tasks/threads would be bad +/// +internal sealed class SyncThreadPool : IDisposable +{ + private readonly Channel> _workQueue; + private readonly Task[] _workers; + private readonly CancellationTokenSource _cts = new(); + + public SyncThreadPool(int threadCount = 0) + { + // Default to number of physical cores if not specified + if (threadCount <= 0) + threadCount = Environment.ProcessorCount; + + _workQueue = Channel.CreateUnbounded>(); + _workers = new Task[threadCount]; + + for (int i = 0; i < threadCount; i++) + { + _workers[i] = Task + .Factory.StartNew( + Worker, + _cts.Token, + TaskCreationOptions.LongRunning, + TaskScheduler.Default + ) + .Unwrap(); + } + } + + private async Task Worker() + { + try + { + await foreach (var work in _workQueue.Reader.ReadAllAsync(_cts.Token)) + { + await work(_cts.Token); + } + } + catch (OperationCanceledException) { } + catch (Exception ex) + { + // Log the exception or handle it as needed + await Console.Error.WriteLineAsync($"Worker encountered an error: {ex.Message}"); + Environment.Exit(1); + } + } + + public Task RunAsync( + Func operation, + CancellationToken cancellationToken = default + ) + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + _workQueue.Writer.TryWrite(poolCancellationToken => + { + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + poolCancellationToken, + cancellationToken + ); + try + { + var result = operation(linkedCts.Token); + tcs.SetResult(result); + } + catch (Exception ex) + { + tcs.SetException(ex); + } + return Task.CompletedTask; + }); + + return tcs.Task; + } + + public Task RunAsync( + Func> operation, + CancellationToken cancellationToken = default + ) + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + _workQueue.Writer.TryWrite(async poolCancellationToken => + { + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + poolCancellationToken, + cancellationToken + ); + try + { + var result = await operation(linkedCts.Token); + tcs.SetResult(result); + } + catch (Exception ex) + { + tcs.SetException(ex); + } + }); + + return tcs.Task; + } + + public void Dispose() + { + _workQueue.Writer.Complete(); + _cts.Cancel(); + Task.WaitAll(_workers); + _cts.Dispose(); + } +} diff --git a/AppsAnalysis/src/Altinn.Analysis/ValueColumn.cs b/AppsAnalysis/src/Altinn.Analysis/ValueColumn.cs new file mode 100644 index 00000000..13381bed --- /dev/null +++ b/AppsAnalysis/src/Altinn.Analysis/ValueColumn.cs @@ -0,0 +1,16 @@ +using Spectre.Console; +using Spectre.Console.Rendering; + +namespace Altinn.Analysis; + +internal sealed class ValueProgressColumn : ProgressColumn +{ + public override IRenderable Render( + RenderOptions options, + ProgressTask task, + TimeSpan deltaTime + ) => + new Markup($"{task.Value}/{task.MaxValue}") + .Overflow(Overflow.Ellipsis) + .Justify(Justify.Right); +}