Утечка памяти, которую не устраняет GC в отношении .Net Entity Framework версии 8 (консольное приложение C# .Net8.02)

Для большинства вещей, которые я пишу, меня обычно не волнует использование памяти, однако у меня есть консольное приложение, написанное на .Net8.02 с использованием EF8, которое теряет 2 МБ каждый раз при вызове транзакции EF. Я испробовал все мыслимые варианты использования GC (сборщика мусора), пытаясь заставить .Net освобождать память между вызовами, но утечка сохраняется. Я также разместил фрагменты кода во всех местах для отслеживания использования памяти, и все это возвращается в EF. 2 МБ может показаться не таким уж большим, но ежедневно эта программа считывает около 5000 файлов журналов (или более), и программа приводит к сбою машины (VM или реальной) - я написал монитор памяти с GC, чтобы посмотреть доступную память, а затем остановиться обработки и подождите, пока он снова не отключится (это никогда не происходит!).

В любом случае, именно в этом и заключается проблема (LogFileProcessor.cs). Если у кого-нибудь есть идеи о том, как освободить память, которую поглощает EF, я был бы очень признателен (прежде чем я вернусь и буду использовать ADO.Net Core (у которого нет такой же проблемы - разберитесь)

 public partial class LogFileProcessor(LogDbContext dbContext, ILogger<LogFileProcessor> logger)
 {
     private readonly LogDbContext _dbContext = dbContext;
     private readonly ILogger<LogFileProcessor> _logger = logger;

     private List<LogEntry> _logEntries = new List<LogEntry>();
     private string[] _lines = [];

     public async Task<bool> ProcessLogFileAsync(string filePath)
     {
         bool result = false;

         if (!File.Exists(filePath))
         {
             _logger.LogError("File not found: {filePath}", filePath);
             return result;
         }

         string fileName = Path.GetFileName(filePath);

         if (await LogAlreadyProcessedAsync(fileName))
         {
             _logger.LogInformation("Log file already processed: {fileName}", fileName);
             return result;
         }

         string fileNameNoExt = Path.GetFileNameWithoutExtension(filePath);
         DateTime fileDate = File.GetLastWriteTime(filePath);
         string fileHash = ProgramBase.ComputeSha256Hash(filePath);
         int logFileId = ExtractLogFileId(fileNameNoExt);
         string fileType = ExtractFileType(fileNameNoExt);

         using var transaction = await _dbContext.Database.BeginTransactionAsync();
         try
         {
             var parsedLog = new ParsedLog
             {
                 FileName = fileName,
                 LogType = fileType,
                 LogFileId = logFileId,
                 DateParsed = DateTime.UtcNow,
                 FileDate = fileDate,
                 FileHash = fileHash
             };

             await _dbContext.ParsedLogs.AddAsync(parsedLog);
             await _dbContext.SaveChangesAsync();

             int parsedLogId = parsedLog.Id; //retrieve new Id (identity) from ParsedLogs table

             //_lines = await File.ReadLinesAsync(filePath).ToArray();   //not really needed, but if user sets log file size really large, this is better for resources
             _lines = await File.ReadAllLinesAsync(filePath);
             int lineNum = 0;
             foreach (var line in _lines)
             {
                 var entry = ParseLine(line, parsedLogId, lineNum);
                 if (entry != null)
                 {
                     _logEntries.Add(entry);
                 }
                 else
                 {
                     throw new Exception($"Unable to parse or convert line {lineNum}");
                 }
                 lineNum += 1;
             }

             await _dbContext.LogEntries.AddRangeAsync(_logEntries);
             await _dbContext.SaveChangesAsync();

             await transaction.CommitAsync();
             _logger.LogInformation("Log file: {fileName} processed and data committed to the database.", fileName);
             await transaction.DisposeAsync();
             result = true;
         }
         catch (Exception ex)
         {
             await transaction.RollbackAsync();
             _logger.LogError("Error processing log file: {fileName} {ex.Message}", fileName, ex.Message);
             await transaction.DisposeAsync();
             result = false;
         }
         finally
         {
             _logEntries.Clear();
             _lines = [];

             // Force garbage collection - naturally, this doesn't work, UGH!
             GC.Collect();
             GC.WaitForPendingFinalizers();
             GC.Collect();
         }
         return result;
     }



     private async Task<bool> LogAlreadyProcessedAsync(string fileName)
     {
         return await _dbContext.ParsedLogs.AsNoTracking().AnyAsync(l => l.FileName == fileName);
     }

     private static string ExtractFileType(string fileNameNoExt)
     {
         var match = FileTypeRegex().Match(fileNameNoExt);
         return match.Success ? match.Groups[1].Value : "unknown";
     }

     private static int ExtractLogFileId(string fileNameNoExt)
     {
         var match = FileIdRegex().Match(fileNameNoExt);
         return match.Success ? int.Parse(match.Groups[1].Value) : 0;
     }


     private static LogEntry? ParseLine(string line, int parsedLogId, int lineNum)
     {
         var parts = line.Split("->", StringSplitOptions.TrimEntries);
         if (parts.Length < 2) return null;

         var dateTimePart = parts[0].Trim();
         string ipPart = string.Empty;
         string statusAndRestPart;

         // Check if the IP address is present
         if (parts.Length == 3)
         {
             ipPart = parts[1].Trim();
             statusAndRestPart = parts[2].Trim();
         }
         else
         {
             // Assume the IP address is missing and adjust accordingly
             statusAndRestPart = parts[1].Trim();
         }

         var statusPart = statusAndRestPart.Split(':', StringSplitOptions.TrimEntries)[0];
         var actionDetailsPart = ActionDetailsRegex().Match(statusAndRestPart);

         string action = actionDetailsPart.Groups[1].Value.Trim();
         string details = actionDetailsPart.Groups.Count > 2 ? actionDetailsPart.Groups[2].Value.Trim() : string.Empty;

         return new LogEntry
         {
             ParsedLogId = parsedLogId,
             LineNum = lineNum,
             EntryDate = DateTime.ParseExact(dateTimePart, "ddd, dd MMM yyyy HH:mm:ss", CultureInfo.InvariantCulture),
             IPaddress = ipPart,
             Status = statusPart,
             Action = action,
             Details = details
         };
     }

     // generates all regexes at compile time
     [GeneratedRegex(@"^(.*?)_\d+$")]
     private static partial Regex FileTypeRegex();

     [GeneratedRegex(@"_([0-9]+)$")]
     private static partial Regex FileIdRegex();

     [GeneratedRegex(@"Action=\[(.*?)\](?:, Details=\[(.*?)\])?", RegexOptions.Compiled)]
     private static partial Regex ActionDetailsRegex();
 }

Файл program.cs:

    namespace LogParserApp;

using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Configuration;
using System;
using System.IO;
using Microsoft.EntityFrameworkCore;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;

internal partial class Program : ProgramBase
{
    public static async Task Main(string[] args)
    {
        var settings = ParseArguments(args);

        if (!settings.TryGetValue("filetype", out List<string>? value) || value.Count == 0)
        {
            Console.WriteLine("Please specify at least one filetype using '-filetype \"smtp, pop3\"'.");
            return;
        }

        var host = CreateHostBuilder(args).Build();

        // Access the configuration and the LogFileProcessor service
        var config = host.Services.GetRequiredService<IConfiguration>();

        string? folderPath = settings.TryGetValue("folderpath", out List<string>? value1) && value1.Count > 0 ? value1[0]
                              : config["LogFileSettings:FolderPath"];

        string? archivePath = settings.TryGetValue("archivepath", out List<string>? value2) && value2.Count > 0 ? value2[0]
                              : config["LogFileSettings:ArchivePath"];

        var logFileProcessor = host.Services.GetRequiredService<LogFileProcessor>();

        string postProcess = settings.TryGetValue("postprocess", out List<string>? value3) && value3.Count > 0 ? value3[0].ToLower() : "keep";


        foreach (var fileType in value)
        {
            var logFiles = Directory.GetFiles(folderPath ?? "C:\\logs", $"{fileType}_*.txt")
                .Select(file => new
                {
                    FileName = file,
                    OrderKey = int.Parse(OrderKeyRegex().Match(Path.GetFileName(file)).Groups[1].Value)
                })
                .OrderBy(f => f.OrderKey)
                .Select(f => f.FileName);

            //long memOffset = GC.GetTotalMemory(forceFullCollection: true); //for tracking memory

            foreach (var file in logFiles)
            {
                // EnsureAvailableMemory(); //to keep program from crashing, no joy

                //long startMem = GC.GetTotalMemory(forceFullCollection: true);  //for tracking memory

                Console.WriteLine($"Processing file: {file}");
                var processSuccess = (await logFileProcessor.ProcessLogFileAsync(file));

                if (processSuccess)
                {
                    switch (postProcess)
                    {
                        case "archive":
                            string targetPath = Path.Combine(archivePath ?? "C:\\logs\\archive", Path.GetFileName(file));
                            File.Move(file, targetPath);
                            Console.WriteLine($"Archived file to: {targetPath}");
                            break;
                        case "delete":
                            File.Delete(file);
                            Console.WriteLine($"Deleted file: {file}");
                            break;
                        case "keep":
                            // Nothing to do, may add something later to keep, but rename, or what-have-you
                            break;
                    }
                }
                else
                {
                    Console.WriteLine($"Processing failed for file: {file}, skipping post-processing steps.");
                }

                GC.Collect(0, GCCollectionMode.Forced);

                //long endMem = GC.GetTotalMemory(forceFullCollection: true);  //for tracking memory
                //Console.WriteLine($"Memory Utilized: {(endMem - startMem) / 1048576M:N2} MB");  //for tracking memory
                //Console.WriteLine($"Running Memory: {(endMem - memOffset) / 1048576M:N2} MB");  //for tracking memory

            }
        }

        await host.RunAsync();
    }

    static IHostBuilder CreateHostBuilder(string[] args) =>
    Host.CreateDefaultBuilder(args)
        .ConfigureAppConfiguration((hostingContext, config) =>
        {
            config.SetBasePath(Directory.GetCurrentDirectory());
            config.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true);
        })
        .ConfigureServices((hostContext, services) =>
        {
            services.AddDbContext<LogDbContext>(options =>
                options.UseSqlServer(hostContext.Configuration.GetConnectionString("DefaultConnection")));

            services.AddScoped<LogFileProcessor>();
            services.AddLogging(); 

            services.AddSingleton<IConfiguration>(hostContext.Configuration);
        })
        .ConfigureLogging(logging => {
            logging.ClearProviders();
            logging.AddConsole();
            logging.AddFilter("Microsoft.EntityFrameworkCore.Database.Command", LogLevel.Warning);
        });


    // generates a regex at compile time
    [GeneratedRegex(@"^.*?_(\d+)\.txt$")]
    private static partial Regex OrderKeyRegex();


    // this doesn't help - garbage collection never actually occurs, so it stays at 1GB & tries again indefinitely
    public static void EnsureAvailableMemory()
    {
        const long maxAllowedMemory = 1_073_741_824; // Set threshold to 1 GB

        while (true)
        {
            long memoryUsed = GC.GetTotalMemory(false);
            Console.WriteLine($"Memory used: {memoryUsed} bytes");

            if (memoryUsed < maxAllowedMemory)
            {
                break;
            }

            Console.WriteLine("Memory usage is too high, forcing garbage collection.");
            GC.Collect();
            GC.WaitForPendingFinalizers();
            Console.WriteLine("Garbage collection complete, pausing for a few seconds...");
            Thread.Sleep(5000); // Wait 5 seconds before checking again
        }
    }

}

Обратите внимание на последний метод (я пробовал разные вещи с GC, но безуспешно).

Сущности (вероятно не помогут, но вот они)

public class ParsedLog
{
    public int Id { get; set; }
    public string FileName { get; set; } = string.Empty;
    public string LogType { get; set; } = string.Empty;
    public int LogFileId { get; set; }
    public DateTime DateParsed { get; set; }
    public DateTime FileDate { get; set; }
    public string? FileHash { get; set; }  // SHA-256 hash of the file
}

public class LogEntry
{
    public long Id { get; set; }
    public int ParsedLogId { get; set; }
    public int LineNum { get; set; }
    public DateTime EntryDate { get; set; }
    public string IPaddress { get; set; } = string.Empty;
    public string Status { get; set; } = string.Empty;
    public string Action { get; set; } = string.Empty;
    public string Details { get; set; } = string.Empty;

}


public class LogDbContext(DbContextOptions<LogDbContext> options) : DbContext(options)
{
    public DbSet<LogEntry> LogEntries { get; set; }
    public DbSet<ParsedLog> ParsedLogs { get; set; }
}

Я планирую провести рефакторинг для очевидного повышения скорости (предварительное хеширование файлов, использование интервалов, групповая вставка и т. д.), но проблема с памятью становится довольно серьезной при обработке тысяч файлов.

Вот некоторые результаты, демонстрирующие увеличение объема памяти до 1 ГБ по 1-2 МБ за раз.

    PS D:\Projects\LogParserApp> dotnet run -filetype "smtp" -postprocess "archive"

Processing file: D:\EmailLogs\smtp_0.txt
info: LogParserApp.LogFileProcessor[0]
      Log file: smtp_0.txt processed and data committed to the database.
Archived file to: D:\EmailLogs\ArchivedLogs\smtp_0.txt
Memory Utilized: 12.27 MB
Running Memory: 12.49 MB
Processing file: D:\EmailLogs\smtp_1.txt
info: LogParserApp.LogFileProcessor[0]
      Log file: smtp_1.txt processed and data committed to the database.
Archived file to: D:\EmailLogs\ArchivedLogs\smtp_1.txt
Memory Utilized: 2.78 MB
Running Memory: 15.27 MB
Processing file: D:\EmailLogs\smtp_2.txt
info: LogParserApp.LogFileProcessor[0]
      Log file: smtp_2.txt processed and data committed to the database.
Archived file to: D:\EmailLogs\ArchivedLogs\smtp_2.txt
Memory Utilized: 2.48 MB
Running Memory: 17.74 MB
Processing file: D:\EmailLogs\smtp_3.txt
info: LogParserApp.LogFileProcessor[0]
      Log file: smtp_3.txt processed and data committed to the database.
Archived file to: D:\EmailLogs\ArchivedLogs\smtp_3.txt
Memory Utilized: 3.28 MB
Running Memory: 21.03 MB
Processing file: D:\EmailLogs\smtp_4.txt
info: LogParserApp.LogFileProcessor[0]
      Log file: smtp_4.txt processed and data committed to the database.
Archived file to: D:\EmailLogs\ArchivedLogs\smtp_4.txt
Memory Utilized: 2.28 MB
Running Memory: 23.31 MB
Processing file: D:\EmailLogs\smtp_5.txt
info: LogParserApp.LogFileProcessor[0]
      Log file: smtp_5.txt processed and data committed to the database.
Archived file to: D:\EmailLogs\ArchivedLogs\smtp_5.txt
Memory Utilized: 2.55 MB
Running Memory: 25.86 MB
...
...
...
Processing file: D:\EmailLogs\smtp_370.txt
info: LogParserApp.LogFileProcessor[0]
      Log file: smtp_370.txt processed and data committed to the database.
Archived file to: D:\EmailLogs\ArchivedLogs\smtp_370.txt
Memory Utilized: 2.36 MB
Running Memory: 999.33 MB
Processing file: D:\EmailLogs\smtp_371.txt
info: LogParserApp.LogFileProcessor[0]
      Log file: smtp_371.txt processed and data committed to the database.
Archived file to: D:\EmailLogs\ArchivedLogs\smtp_371.txt
Memory Utilized: 2.59 MB
Running Memory: 1,001.92 MB
Processing file: D:\EmailLogs\smtp_372.txt
info: LogParserApp.LogFileProcessor[0]
      Log file: smtp_372.txt processed and data committed to the database.
Archived file to: D:\EmailLogs\ArchivedLogs\smtp_372.txt
Memory Utilized: 2.24 MB
Running Memory: 1,004.16 MB

Это на 373 файла - представьте себе 10 тысяч файлов. :)