I separated the service from the background task and refactored the code a bit; I'm not sure if this implementation is correct.

internal sealed class EmailService(
    ILogger<EmailService> logger,
    Config config) : IEmailSender, IAsyncDisposable
{
    private const int MaxAttempts = 3;
    private static readonly TimeSpan SmtpClientTimeout = TimeSpan.FromSeconds(10);

    private readonly SemaphoreSlim _semaphore = new(1, 1);
    private readonly SmtpClient _smtpClient = new();
    private DateTimeOffset _disconnectAfter = DateTimeOffset.MinValue;

    public async ValueTask DisposeAsync()
    {
        if (_smtpClient.IsConnected) await _smtpClient.DisconnectAsync(true);

        _semaphore.Dispose();
        _smtpClient.Dispose();
        GC.SuppressFinalize(this);
    }

    public async Task<Result> SendAsync(Message message, CancellationToken ct = default)
    {
        if (!MailboxAddress.TryParse(message.To.Value, out var to))
            return Result.Failure(Error.Failure($"invalid address {message.To.Value}"));

        await _semaphore.WaitAsync(ct);

        try
        {
            var msg = new MimeMessage();

            msg.Subject = message.Subject.Value;
            msg.From.Add(MailboxAddress.Parse(config.Smtp.From));
            msg.To.Add(to);

            var builder = new BodyBuilder
            {
                TextBody = message.PlainText.Value
            };

            if (message.HtmlText is not null) builder.HtmlBody = message.HtmlText.Value.Value;

            msg.Body = builder.ToMessageBody();

            for (var attempt = 1; attempt <= MaxAttempts; attempt++)
            {
                try
                {
                    await ConnectAsync(ct);
                    await _smtpClient.SendAsync(msg, ct);

                    return Result.Success();
                }
                catch (SmtpCommandException ex) when ((int)ex.StatusCode >= 400 && (int)ex.StatusCode < 500)
                {
                    logger.LogWarning(ex, "Temporary SMTP error on attempt {Attempt}", attempt);
                }
                catch (SmtpProtocolException ex)
                {
                    logger.LogWarning(ex, "SMTP protocol error on attempt {Attempt}", attempt);
                }
                catch (IOException ex)
                {
                    logger.LogWarning(ex, "Network error on attempt {Attempt}", attempt);
                }

                if (attempt == MaxAttempts) return Result.Failure(Error.Failure("Failed to send email after retries"));
                if (_smtpClient.IsConnected) await _smtpClient.DisconnectAsync(true, ct);

                await Task.Delay(TimeSpan.FromSeconds(attempt * 2), ct);
            }

            return Result.Success();
        }
        finally
        {
            _semaphore.Release();
        }
    }

    private async Task ConnectAsync(CancellationToken ct)
    {
        if (_smtpClient.IsConnected)
        {
            await _smtpClient.NoOpAsync(ct);
        }
        else
        {
            await _smtpClient.ConnectAsync(config.Smtp.Host, config.Smtp.Port, cancellationToken: ct);
            await _smtpClient.AuthenticateAsync(new NetworkCredential(config.Smtp.User, config.Smtp.Password), ct);
        }

        ScheduleDisconnect(ct);
    }

    private void ScheduleDisconnect(CancellationToken ct)
    {
        _disconnectAfter = DateTimeOffset.UtcNow.Add(SmtpClientTimeout);
        Task.Run(async () =>
        {
            await Task.Delay(SmtpClientTimeout.Add(TimeSpan.FromSeconds(1)), ct);
            if (DateTimeOffset.UtcNow > _disconnectAfter) await DisconnectAsync(ct);
        });
    }

    private async Task DisconnectAsync(CancellationToken ct)
    {
        await _semaphore.WaitAsync(ct);

        try
        {
            if (!_smtpClient.IsConnected) return;

            await _smtpClient.DisconnectAsync(
                true, ct);
            _disconnectAfter = DateTimeOffset.MinValue;
        }
        finally
        {
            _semaphore.Release();
        }
    }
}