Added submissions archives.

Common/OJS.Common/Extensions/EnumerableExtensions.cs

@@ -5,7 +5,14 @@ namespace OJS.Common.Extensions;
 
 public static class EnumerableExtensions
 {
-    public static T? MaxOrDefault<T>(this IEnumerable<T> enumerable)
-        => enumerable.DefaultIfEmpty()
-            .Max();
+    public static IEnumerable<IEnumerable<T>> InBatches<T>(this IEnumerable<T> queryable, int size)
+    {
+        var current = queryable.AsQueryable();
+        while (current.Any())
+        {
+            var batch = current.Take(size);
+            yield return batch;
+            current = current.Skip(size);
+        }
+    }
 }

Data/OJS.Data.Models/Submissions/ArchivedSubmission.cs

@@ -0,0 +1,77 @@
+namespace OJS.Data.Models.Submissions
+{
+    using System;
+    using System.ComponentModel.DataAnnotations;
+    using System.ComponentModel.DataAnnotations.Schema;
+    using AutoMapper;
+    using OJS.Data.Validation;
+    using OJS.Services.Infrastructure.Models.Mapping;
+    using OJS.Workers.Common.Models;
+
+    [Table("Submissions")]
+    public class ArchivedSubmission : IMapExplicitly, IEquatable<ArchivedSubmission>
+    {
+        [Key]
+        [DatabaseGenerated(DatabaseGeneratedOption.None)]
+        public int Id { get; set; }
+
+        public int ParticipantId { get; set; }
+
+        public int ProblemId { get; set; }
+
+        public int? SubmissionTypeId { get; set; }
+
+        public byte[] Content { get; set; } = [];
+
+        public string? FileExtension { get; set; }
+
+        public byte[]? SolutionSkeleton { get; set; }
+
+        public DateTime? StartedExecutionOn { get; set; }
+
+        public DateTime? CompletedExecutionOn { get; set; }
+
+        [StringLength(ConstraintConstants.IpAddressMaxLength)]
+        [Column(TypeName = "varchar")]
+        public string? IpAddress { get; set; }
+
+        [StringLength(ConstraintConstants.Submission.WorkerNameMaxLength)]
+        public string? WorkerName { get; set; }
+
+        public ExceptionType? ExceptionType { get; set; }
+
+        public bool Processed { get; set; }
+
+        public int Points { get; set; }
+
+        public string? ProcessingComment { get; set; }
+
+        public string? TestRunsCache { get; set; }
+
+        public DateTime CreatedOn { get; set; }
+
+        public DateTime? ModifiedOn { get; set; }
+
+        public bool IsHardDeletedFromMainDatabase { get; set; }
+
+        [NotMapped]
+        public bool IsBinaryFile => !string.IsNullOrWhiteSpace(this.FileExtension);
+
+        [NotMapped]
+        public string ContentAsString
+            => this.IsBinaryFile ? string.Empty : this.Content.ToString();
+
+        public override bool Equals(object? obj)
+            => obj is ArchivedSubmission other && this.Equals(other);
+
+        public bool Equals(ArchivedSubmission? other)
+            => other != null && this.Id == other.Id;
+
+        public override int GetHashCode()
+            => this.Id.GetHashCode();
+
+        public void RegisterMappings(IProfileExpression configuration)
+            => configuration.CreateMap<Submission, ArchivedSubmission>()
+                .ForMember(d => d.IsHardDeletedFromMainDatabase, opt => opt.MapFrom(s => false));
+    }
+}

Data/OJS.Data/ArchivesDbContext.cs

@@ -0,0 +1,9 @@
+namespace OJS.Data;
+
+using Microsoft.EntityFrameworkCore;
+using OJS.Data.Models.Submissions;
+
+public class ArchivesDbContext(DbContextOptions<ArchivesDbContext> options) : DbContext(options)
+{
+    public DbSet<ArchivedSubmission> Submissions { get; set; } = null!;
+}

Servers/Administration/OJS.Servers.Administration/Extensions/ServiceCollectionExtensions.cs

@@ -40,12 +40,14 @@ public static void ConfigureServices(
             .AddHttpClients(configuration)
             .AddTransient(typeof(IDataService<>), typeof(AdministrationDataService<>))
             .AddTransient<ITransactionsProvider, TransactionsProvider<OjsDbContext>>()
+            .AddTransient<IArchivesDataService, ArchivesDataService>()
             .AddTransient<AdministrationExceptionMiddleware>()
             .AddHangfireServer(configuration, AppName, [AdministrationQueueName])
             .AddMessageQueue<Program>(configuration)
             .ConfigureGlobalDateFormat()
             .ConfigureCorsPolicy(configuration)
             .AddIdentityDatabase<OjsDbContext, UserProfile, Role, UserInRole>(configuration)
+            .AddArchivesDatabase(configuration)
             .AddResiliencePipelines()
             .AddMemoryCache()
             .AddDistributedCaching(configuration)

Servers/Administration/OJS.Servers.Administration/Program.cs

@@ -1,6 +1,7 @@
 namespace OJS.Servers.Administration;
 
 using Microsoft.AspNetCore.Builder;
+using OJS.Data;
 using OJS.Servers.Administration.Extensions;
 using OJS.Servers.Infrastructure.Extensions;
 
@@ -16,6 +17,8 @@ public static void Main(string[] args)
 
         var app = builder.Build();
 
+        app.MigrateDatabase<ArchivesDbContext>(builder.Configuration);
+
         app.ConfigureWebApplication(builder.Configuration);
         app.Run();
     }

Servers/Infrastructure/OJS.Servers.Infrastructure/Extensions/ServiceCollectionExtensions.cs

@@ -54,6 +54,8 @@ namespace OJS.Servers.Infrastructure.Extensions
     using System.Security.Claims;
     using System.Text.Json;
     using System.Threading.Tasks;
+    using Microsoft.Data.SqlClient;
+    using OJS.Data;
     using OpenAI;
     using RabbitMQ.Client;
     using static OJS.Common.GlobalConstants;
@@ -199,6 +201,35 @@ public static IServiceCollection AddHangfireServer(
             return services;
         }
 
+        /// <summary>
+        /// Adds the archives database context to the service collection.
+        /// </summary>
+        /// <param name="services">The service collection.</param>
+        /// <param name="configuration">The configuration.</param>
+        public static IServiceCollection AddArchivesDatabase(
+            this IServiceCollection services,
+            IConfiguration configuration)
+        {
+            var defaultConnectionString = configuration.GetConnectionString(DefaultDbConnectionName);
+
+            // Modify the connection string to use a different database name for archives
+            var builder = new SqlConnectionStringBuilder(defaultConnectionString);
+            builder.InitialCatalog = $"{builder.InitialCatalog}Archives";
+            var connectionString = builder.ConnectionString;
+
+            services
+                .AddDbContext<ArchivesDbContext>(options =>
+                {
+                    options.UseSqlServer(connectionString);
+                });
+
+            services
+                .AddHealthChecks()
+                .AddSqlServer(connectionString, name: "archives-db");
+
+            return services;
+        }
+
         public static IServiceCollection AddSwaggerDocs(
             this IServiceCollection services,
             string name,

Servers/Infrastructure/OJS.Servers.Infrastructure/OJS.Servers.Infrastructure.csproj

@@ -4,6 +4,7 @@
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\..\Common\OJS.Common\OJS.Common.csproj" />
+    <ProjectReference Include="..\..\..\Data\OJS.Data\OJS.Data.csproj" />
     <ProjectReference Include="..\..\..\Services\Common\OJS.Services.Common\OJS.Services.Common.csproj" />
     <ProjectReference Include="..\..\..\Services\Infrastructure\OJS.Services.Infrastructure\OJS.Services.Infrastructure.csproj" />
   </ItemGroup>

Services/Administration/OJS.Services.Administration.Business/Implementations/ArchivedSubmissionsBusinessService.cs

@@ -0,0 +1,116 @@
+namespace OJS.Services.Administration.Business.Implementations;
+
+using System.Linq;
+using System.Threading.Tasks;
+using DocumentFormat.OpenXml.Vml;
+using OJS.Common;
+using OJS.Data.Models.Submissions;
+using OJS.Services.Administration.Data;
+using OJS.Services.Common;
+using OJS.Services.Common.Data;
+using OJS.Services.Infrastructure;
+using OJS.Services.Infrastructure.Extensions;
+
+public class ArchivedSubmissionsBusinessService : IArchivedSubmissionsBusinessService
+{
+    private readonly ISubmissionsDataService submissionsData;
+    private readonly IArchivesDataService archivesData;
+    private readonly IDatesService dates;
+
+    public ArchivedSubmissionsBusinessService(
+        ISubmissionsDataService submissionsData,
+        IArchivesDataService archivesData,
+        IDatesService dates)
+    {
+        this.submissionsData = submissionsData;
+        this.archivesData = archivesData;
+        this.dates = dates;
+    }
+
+    public async Task<int> ArchiveOldSubmissionsDailyBatch(int limit, int maxSubBatchSize)
+    {
+        var leftoverSubmissionsFromBatchSplitting = limit % maxSubBatchSize;
+        var numberOfIterations = limit / maxSubBatchSize;
+        if(leftoverSubmissionsFromBatchSplitting > 0)
+        {
+            numberOfIterations++;
+        }
+
+        var archived = 0;
+
+        for (var i = 0; i < numberOfIterations; i++)
+        {
+            var curBatchSize = maxSubBatchSize;
+            var isLastIteration = i == (numberOfIterations - 1);
+            if(leftoverSubmissionsFromBatchSplitting > 0 && isLastIteration)
+            {
+                curBatchSize = leftoverSubmissionsFromBatchSplitting;
+            }
+
+            var allSubmissionsForArchive = this
+                .GetSubmissionsForArchiving()
+                .OrderBy(x => x.Id)
+                .InBatches(GlobalConstants.BatchOperationsChunkSize, curBatchSize);
+
+            foreach (var submissionsForArchiveBatch in allSubmissionsForArchive)
+            {
+                var submissionsForArchives = submissionsForArchiveBatch
+                    .MapCollection<ArchivedSubmission>()
+                    .ToList();
+
+                if(submissionsForArchives.Count == 0)
+                {
+                    break;
+                }
+
+                archived += await this.archivesData.AddMany(submissionsForArchives);
+                await this.archivesData.SaveChanges();
+            }
+
+            await this.submissionsData.HardDeleteArchived(curBatchSize);
+        }
+
+        return archived;
+    }
+
+    public async Task<int> ArchiveOldSubmissionsWithLimit(int limit)
+    {
+        var archived = 0;
+        var allSubmissionsForArchive = this
+            .GetSubmissionsForArchiving()
+            .OrderBy(x => x.Id)
+            .InBatches(GlobalConstants.BatchOperationsChunkSize, limit);
+
+        foreach (var submissionsForArchiveBatch in allSubmissionsForArchive)
+        {
+            var submissionsForArchives = submissionsForArchiveBatch
+                .MapCollection<ArchivedSubmission>()
+                .ToList();
+
+            if(submissionsForArchives.Count == 0)
+            {
+                break;
+            }
+
+            archived += await this.archivesData.AddMany(submissionsForArchives);
+            await this.archivesData.SaveChanges();
+        }
+
+        return archived;
+    }
+
+    public async Task<int> HardDeleteArchivedByLimit(int limit)
+        => await this.submissionsData.HardDeleteArchived(limit);
+
+    private IQueryable<Submission> GetSubmissionsForArchiving()
+    {
+        var now = this.dates.GetUtcNow();
+        var bestSubmissionCutoffDate = now.AddYears(-GlobalConstants.BestSubmissionEligibleForArchiveAgeInYears);
+        var nonBestSubmissionCutoffDate = now.AddYears(-GlobalConstants.NonBestSubmissionEligibleForArchiveAgeInYears);
+
+        return this.submissionsData
+            .GetAllCreatedBeforeDateAndNonBestCreatedBeforeDate(
+                bestSubmissionCutoffDate,
+                nonBestSubmissionCutoffDate);
+    }
+}

Services/Administration/OJS.Services.Administration.Business/Implementations/RecurringBackgroundJobsBusinessService.cs

@@ -15,19 +15,22 @@ public class RecurringBackgroundJobsBusinessService : IRecurringBackgroundJobsBu
         private readonly ISubmissionsForProcessingBusinessService submissionsForProcessing;
         private readonly IParticipantsBusinessService participantsBusinessService;
         private readonly IParticipantScoresBusinessService participantScoresBusiness;
+        private readonly IArchivedSubmissionsBusinessService archivedSubmissionsBusiness;
         private readonly IBusControl bus;
         private readonly ILogger<RecurringBackgroundJobsBusinessService> logger;
 
         public RecurringBackgroundJobsBusinessService(
             ISubmissionsForProcessingBusinessService submissionsForProcessing,
             IParticipantsBusinessService participantsBusinessService,
             IParticipantScoresBusinessService participantScoresBusiness,
+            IArchivedSubmissionsBusinessService archivedSubmissionsBusiness,
             IBusControl bus,
             ILogger<RecurringBackgroundJobsBusinessService> logger)
         {
             this.submissionsForProcessing = submissionsForProcessing;
             this.participantsBusinessService = participantsBusinessService;
             this.participantScoresBusiness = participantScoresBusiness;
+            this.archivedSubmissionsBusiness = archivedSubmissionsBusiness;
             this.bus = bus;
             this.logger = logger;
         }
@@ -77,5 +80,27 @@ public async Task<object> NormalizeAllPointsThatExceedAllowedLimit()
 
             return "Successfully normalized all points that exceed allowed limit";
         }
+
+        public async Task<object> ArchiveOldSubmissionsDailyBatch()
+        {
+            const int archiveDailyBatchLimit = 500_000;
+            const int archiveMaxSubBatchSize = 10_000;
+
+            var archivedCount = await this.archivedSubmissionsBusiness.ArchiveOldSubmissionsDailyBatch(
+                archiveDailyBatchLimit,
+                archiveMaxSubBatchSize);
+
+            return $"Successfully archived {archivedCount} submissions.";
+        }
+
+        public async Task<object> HardDeleteArchivedSubmissions()
+        {
+            const int archiveSingleBatchLimit = 25_000;
+
+            var hardDeletedCount = await this.archivedSubmissionsBusiness.HardDeleteArchivedByLimit(
+                archiveSingleBatchLimit);
+
+            return $"Successfully hard deleted {hardDeletedCount} archived submissions.";
+        }
     }
 }

Services/Administration/OJS.Services.Administration.Data/IParticipantScoresDataService.cs

@@ -29,5 +29,7 @@ Task UpdateBySubmissionAndPoints(
             int submissionPoints,
             Participant participant,
             bool shouldSaveChanges = true);
+
+        Task RemoveSubmissionIdsBySubmissionIds(IEnumerable<int> submissionIds);
     }
 }

Services/Administration/OJS.Services.Administration.Data/ISubmissionsDataService.cs

@@ -40,5 +40,7 @@ IQueryable<Submission> GetAllCreatedBeforeDateAndNonBestCreatedBeforeDate(
         Task RemoveTestRunsCacheByProblem(int problemId);
 
         Task<IEnumerable<int>> GetIdsByProblemId(int problemId);
+
+        Task<int> HardDeleteArchived(int deleteCountLimit = 0);
     }
 }