Unable to connect and write logs to Elasticsearch v8.8.1

[tkardaridis]

I have the following configuration in asp.net core application

builder.ConfigureLogging((ctx, loggingBuilder) =>
{
loggingBuilder.ClearProviders();

            if (ctx.HostingEnvironment.IsDevelopment())
            {
                loggingBuilder.AddDebug();
            }

            //Serilog Setup
            Serilog.Log.Logger = **GetELKLogger**(configuration, ctx.HostingEnvironment.EnvironmentName);
            loggingBuilder.Services.AddLogging(configure =>
            {
                configure.AddSerilog(Serilog.Log.Logger, true);
            });

            loggingBuilder.AddConsole();

            loggingBuilder.Services.Configure<AzureFileLoggerOptions>(options =>
            {
                options.FileName = "az.log";
                options.FileSizeLimit = 5 * 1024;
                options.RetainedFileCountLimit = 3;
            });
        }).UseSerilog();



    private static Serilog.ILogger GetELKLogger(IConfiguration config, string environment)
    {
        var appName = Assembly.GetExecutingAssembly().GetName().Name?.ToLower();
        
        var seriLogger = new LoggerConfiguration()
            .MinimumLevel.Debug()
            .MinimumLevel.Override("Microsoft", LogEventLevel.Information)
            .MinimumLevel.Override("Worker", LogEventLevel.Information)
            .MinimumLevel.Override("Host", LogEventLevel.Information)
            .MinimumLevel.Override("System", LogEventLevel.Information)
            .MinimumLevel.Override("Function", LogEventLevel.Information)
            .MinimumLevel.Override("Azure", LogEventLevel.Information)
            .MinimumLevel.Override("Azure.Storage.Blobs", LogEventLevel.Information)
            .MinimumLevel.Override("Azure.Core", LogEventLevel.Information)
            .MinimumLevel.Override("DurableTask", LogEventLevel.Information)
            .Enrich.WithProperty("Application", appName!)
            .Enrich.WithProperty("Environment", environment)
            .Enrich.FromLogContext()
            .Enrich.WithExceptionDetails()
            .Enrich.WithMachineName()
            .Enrich.WithEnvironmentName()
            .Enrich.WithClientIp()
            .Enrich.WithClientAgent()
            .WriteTo.Debug()
            .WriteTo.Console()
            .WriteTo.ApplicationInsights(ConfigureSerilogApplicationInsights(config), TelemetryConverter.Traces, LogEventLevel.Information)
            .WriteTo.Elasticsearch(ConfigureElasticSink(config, environment, appName))
            .CreateLogger();

        //The reason of adding the SelfLog.Enable(Console.Error);
        //is to print out any errors that happens when writing logs to Elasticsearch.
        //Which helps in troubleshooting.Also you may consider changing it to write into physical file or any other storage.
        SelfLog.Enable(Console.Error);

        return seriLogger;
    }



    private static TelemetryConfiguration ConfigureSerilogApplicationInsights(IConfiguration config)
    {
        string instrumentationKey = config.GetValue<string>($"{RenoptiTrader.Shared.Constants.AppInsightsSettings}:{RenoptiTrader.Shared.Constants.AppInsightsSettings_InstrumentationKey}");

        return new TelemetryConfiguration 
                    { InstrumentationKey = instrumentationKey };
    }



    private static ElasticsearchSinkOptions ConfigureElasticSink(IConfiguration config, string environment, string appName)
    {
        var templateName = config[$"{RenoptiTrader.Shared.Constants.ElasticSearchSettings}:{RenoptiTrader.Shared.Constants.ElasticSearchSettings_TemplateName}"];
        string elasticUrl = config[$"{RenoptiTrader.Shared.Constants.ElasticSearchSettings}:{RenoptiTrader.Shared.Constants.ElasticSearchSettings_Url}"]!;
        string elasticUsername = config[$"{RenoptiTrader.Shared.Constants.ElasticSearchSettings}:{RenoptiTrader.Shared.Constants.ElasticSearchSettings_Username}"]!;
        string elasticPassword = config[$"{RenoptiTrader.Shared.Constants.ElasticSearchSettings}:{RenoptiTrader.Shared.Constants.ElasticSearchSettings_Password}"]!;
        string elasticIndexFormat = $"{appName}-{environment}-{DateTimeOffset.Now.LocalDateTime:yyyy.MM}";

        var elasticSinkOptions = new ElasticsearchSinkOptions(new Uri(elasticUrl))
        {
            IndexFormat = elasticIndexFormat,

            AutoRegisterTemplate = true,
            OverwriteTemplate = true,
            TemplateName = templateName,
            AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7,

            //TypeName needs to be null to write to Elastic 8
            TypeName = null,

            BatchAction = ElasticOpType.Create,

            //DetectElasticsearchVersion = false,

            ModifyConnectionSettings = (settings) =>
            {
                settings.EnableApiVersioningHeader();
                settings.CertificateFingerprint(<CertificateFingerprint>);
                settings.BasicAuthentication(elasticUsername, elasticPassword);
                settings.DeadTimeout(TimeSpan.FromSeconds(300));
                return settings;
            }
        };

        return elasticSinkOptions;
    }

The problem is that I still cannot write to ElasticSearch v8.8.1 and I am getting the following error

The client is unable to verify that the server is Elasticsearch due to an unsuccessful product check call

but I have in the settings the EnableApiVersioningHeader().
Following is the errors from the console. Any help please? Has anyone successfully connect to ESv8 and write logs with this Sink?
I was using the same almost code with ESv7 and working perfectly. After the upgrade to ESv8 I did a few changes to the code in order to support the ESv8 as mentioned in various articles but still I cannot.

I will appreciate any help because this is a production environment also and now we don't have logging.

Regards
Thomas

[2023-06-24T15:48:14.941Z] 2023-06-24T15:48:14.9230965Z Caught exception while performing bulk operation to Elasticsearch: Elasticsearch.Net.ElasticsearchClientException: The client is unable to verify that the server is Elasticsearch due to an unsuccessful product check call. Some functionality may not be compatible if the server is running an unsupported product. Call: Status code unknown from: GET /
[2023-06-24T15:48:14.943Z] ---> Elasticsearch.Net.PipelineException: The client is unable to verify that the server is Elasticsearch due to an unsuccessful product check call. Some functionality may not be compatible if the server is running an unsupported product.
[2023-06-24T15:48:14.944Z] ---> System.Threading.Tasks.TaskCanceledException: The request was canceled due to the configured HttpClient.Timeout of 5 seconds elapsing.
[2023-06-24T15:48:14.945Z] ---> System.TimeoutException: The operation was canceled.
[2023-06-24T15:48:14.947Z] ---> System.Threading.Tasks.TaskCanceledException: The operation was canceled.
[2023-06-24T15:48:14.948Z] ---> System.IO.IOException: The response ended prematurely.
[2023-06-24T15:48:14.949Z] at System.Net.Http.HttpConnection.SendAsyncCore(HttpRequestMessage request, Boolean async, CancellationToken cancellationToken)
[2023-06-24T15:48:14.950Z] --- End of inner exception stack trace ---
[2023-06-24T15:48:14.952Z] at System.Net.Http.HttpConnection.SendAsyncCore(HttpRequestMessage request, Boolean async, CancellationToken cancellationToken)
[2023-06-24T15:48:14.955Z] at System.Net.Http.HttpConnectionPool.SendWithVersionDetectionAndRetryAsync(HttpRequestMessage request, Boolean async, Boolean doRequestAuth, CancellationToken cancellationToken)
[2023-06-24T15:48:14.956Z] at System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, Boolean async, CancellationToken cancellationToken)
[2023-06-24T15:48:14.958Z] at System.Net.Http.HttpClient.g__Core|83_0(HttpRequestMessage request, HttpCompletionOption completionOption, CancellationTokenSource cts, Boolean disposeCts, CancellationTokenSource pendingRequestsCts, CancellationToken originalCancellationToken)
[2023-06-24T15:48:14.959Z] --- End of inner exception stack trace ---
[2023-06-24T15:48:14.960Z] --- End of inner exception stack trace ---
[2023-06-24T15:48:14.961Z] at System.Net.Http.HttpClient.HandleFailure(Exception e, Boolean telemetryStarted, HttpResponseMessage response, CancellationTokenSource cts, CancellationToken cancellationToken, CancellationTokenSource pendingRequestsCts)
[2023-06-24T15:48:14.962Z] at System.Net.Http.HttpClient.g__Core|83_0(HttpRequestMessage request, HttpCompletionOption completionOption, CancellationTokenSource cts, Boolean disposeCts, CancellationTokenSource pendingRequestsCts, CancellationToken originalCancellationToken)
[2023-06-24T15:48:14.963Z] at Elasticsearch.Net.HttpConnection.RequestAsync[TResponse](RequestData requestData, CancellationToken cancellationToken)
[2023-06-24T15:48:14.965Z] --- End of inner exception stack trace ---
[2023-06-24T15:48:14.969Z] at Elasticsearch.Net.RequestPipeline.ThrowIfTransientProductCheckFailure()
[2023-06-24T15:48:14.970Z] at Elasticsearch.Net.RequestPipeline.PingAsync(Node node, CancellationToken cancellationToken)
[2023-06-24T15:48:14.971Z] at Elasticsearch.Net.Transport1.PingAsync(IRequestPipeline pipeline, Node node, CancellationToken cancellationToken) [2023-06-24T15:48:14.972Z] at Elasticsearch.Net.Transport1.RequestAsync[TResponse](HttpMethod method, String path, CancellationToken cancellationToken, PostData data, IRequestParameters requestParameters)
[2023-06-24T15:48:14.973Z] --- End of inner exception stack trace ---
[2023-06-24T15:48:14.974Z] at Elasticsearch.Net.Transport1.HandleElasticsearchClientException(RequestData data, Exception clientException, IElasticsearchResponse response) [2023-06-24T15:48:14.975Z] at Elasticsearch.Net.Transport1.FinalizeResponse[TResponse](RequestData requestData, IRequestPipeline pipeline, List1 seenExceptions, TResponse response) [2023-06-24T15:48:14.976Z] at Elasticsearch.Net.Transport1.RequestAsync[TResponse](HttpMethod method, String path, CancellationToken cancellationToken, PostData data, IRequestParameters requestParameters)
[2023-06-24T15:48:14.978Z] at Serilog.Sinks.Elasticsearch.BatchedElasticsearchSink.EmitBatchAsync(IEnumerable`1 events)

[tkardaridis]

The Elasticsearch installation is on a Windows 2019 server using the Windows .zip archive directly instead of using docker container.

[Kitkevich]

Is there any solution? I have the same problem

[mschaefer-gresham]

@tkardaridis not sure if this is exactly the same issue, but I was able to get logging working in 8.11.2. See this issue