using Dpz.Core.MessageQueue.Abstractions;
using Dpz.Core.MessageQueue.Models;
using Medallion.Threading;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using ZiggyCreatures.Caching.Fusion;

namespace Dpz.Core.MessageQueue.RabbitMQ;

/// <summary>
/// Outbox 消息补发后台服务,负责周期性扫描发布失败或消费失败的消息并重新投递
/// </summary>
/// <param name="retryService">Outbox 补发执行器</param>
/// <param name="fusionCache">用于记录上次补发完成状态的 FusionCache 实例</param>
/// <param name="distributedLockProvider">用于多实例互斥补发的分布式锁提供器</param>
/// <param name="logger">日志记录器</param>
public sealed class MessageOutboxRetryBackgroundService(
    IMessageOutboxRetryService retryService,
    IFusionCache fusionCache,
    IDistributedLockProvider distributedLockProvider,
    ILogger<MessageOutboxRetryBackgroundService> logger
) : BackgroundService
{
    /// <summary>
    /// 每轮补发的批次大小
    /// </summary>
    private const int BatchSize = 50;

    /// <summary>
    /// 缓存中保存上次补发状态的键名
    /// </summary>
    private const string StateCacheKey = "Dpz.Core.MessageQueue.OutboxRetry.LastState";

    /// <summary>
    /// 分布式锁资源键名
    /// </summary>
    private const string LockKey = "Dpz.Core.MessageQueue.OutboxRetry.RunLock";

    /// <summary>
    /// 两轮补发之间的最小间隔
    /// </summary>
    private static readonly TimeSpan RetryInterval = TimeSpan.FromMinutes(1);

    /// <summary>
    /// 补发状态在缓存中的保留时长
    /// </summary>
    private static readonly TimeSpan StateCacheDuration = TimeSpan.FromDays(7);

    /// <summary>
    /// 等待获取分布式锁的最大时长
    /// </summary>
    private static readonly TimeSpan LockWaitTimeout = TimeSpan.FromSeconds(3);

    /// <summary>
    /// 本地记录的上次补发完成时间,用于与缓存交叉校验
    /// </summary>
    private DateTimeOffset? _lastCompletedAtUtc;

    /// <summary>
    /// 启动 Outbox 补发循环
    /// </summary>
    /// <param name="stoppingToken">宿主停止信号</param>
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        await LogLastRetryStateAsync(stoppingToken);
        logger.LogInformation("Outbox 重试后台服务已启动");

        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                var delay = await GetDelayBeforeNextRunAsync(stoppingToken);
                if (delay > TimeSpan.Zero)
                {
                    await DelayAsync(delay, stoppingToken);
                    continue;
                }

                await RunOnceAsync(stoppingToken);
            }
            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
            {
                break;
            }
            catch (Exception ex)
            {
                logger.LogError(ex, "Outbox 重试后台服务执行异常");
                await DelayAsync(RetryInterval, stoppingToken);
            }
        }

        logger.LogInformation("Outbox 重试后台服务已停止");
    }

    /// <summary>
    /// 执行单轮 Outbox 补发:获取分布式锁后调用 <see cref="IMessageOutboxRetryService.RetryAsync"/> 并保存状态
    /// </summary>
    /// <param name="cancellationToken">取消令牌</param>
    private async Task RunOnceAsync(CancellationToken cancellationToken)
    {
        await using var lockHandle = await TryAcquireRetryLockAsync(cancellationToken);
        if (lockHandle == null)
        {
            logger.LogInformation("未获取到Outbox重试锁,跳过本轮补发");
            _lastCompletedAtUtc = DateTimeOffset.UtcNow;
            return;
        }

        // 获取锁后再次检查时间,避免多实例在锁外使用同一份过期的缓存状态
        var delay = await GetDelayBeforeNextRunAsync(cancellationToken);
        if (delay > TimeSpan.Zero)
        {
            return;
        }

        var startedAtUtc = DateTimeOffset.UtcNow;
        var result = await retryService.RetryAsync(BatchSize, cancellationToken);
        var completedAtUtc = DateTimeOffset.UtcNow;

        await SaveRetryStateAsync(startedAtUtc, completedAtUtc, result, cancellationToken);

        if (result.TotalRetryCount == 0)
        {
            logger.LogDebug(
                "Outbox 本轮无待重试消息,CompletedAtUtc={CompletedAtUtc}",
                completedAtUtc
            );
            return;
        }

        logger.LogInformation(
            "Outbox 重试完成: Publish={PublishCount}, PublishFailed={PublishFailed}, Consume={ConsumeCount}, ConsumeFailed={ConsumeFailed}, CompletedAtUtc={CompletedAtUtc}",
            result.PublishRetryCount,
            result.PublishFailureCount,
            result.ConsumeRetryCount,
            result.ConsumeFailureCount,
            completedAtUtc
        );
    }

    /// <summary>
    /// 将本轮补发结果写入 FusionCache 并更新本地 <see cref="_lastCompletedAtUtc"/> 字段
    /// </summary>
    /// <param name="startedAtUtc">补发开始时间</param>
    /// <param name="completedAtUtc">补发完成时间</param>
    /// <param name="result">补发结果统计</param>
    /// <param name="cancellationToken">取消令牌</param>
    private async Task SaveRetryStateAsync(
        DateTimeOffset startedAtUtc,
        DateTimeOffset completedAtUtc,
        MessageOutboxRetryResult result,
        CancellationToken cancellationToken
    )
    {
        var state = new MessageOutboxRetryState(
            startedAtUtc,
            completedAtUtc,
            result.PublishRetryCount,
            result.PublishFailureCount,
            result.ConsumeRetryCount,
            result.ConsumeFailureCount
        );

        _lastCompletedAtUtc = completedAtUtc;

        await fusionCache.SetAsync(
            StateCacheKey,
            state,
            options => options.SetDuration(StateCacheDuration).SetFailSafe(false),
            token: cancellationToken
        );
    }

    /// <summary>
    /// 尝试获取用于多实例互斥的分布式锁
    /// </summary>
    /// <param name="cancellationToken">取消令牌</param>
    /// <returns>如果获取成功返回锁句柄;失败返回 <see langword="null"/></returns>
    private async Task<IDistributedSynchronizationHandle?> TryAcquireRetryLockAsync(
        CancellationToken cancellationToken
    )
    {
        return await distributedLockProvider.TryAcquireLockAsync(
            LockKey,
            LockWaitTimeout,
            cancellationToken
        );
    }

    /// <summary>
    /// 根据上次补发完成时间和 <see cref="RetryInterval"/> 计算距下一轮还需等待的时长
    /// </summary>
    /// <param name="cancellationToken">取消令牌</param>
    /// <returns>需要等待的时间;若无需等待则返回 <see cref="TimeSpan.Zero"/></returns>
    private async Task<TimeSpan> GetDelayBeforeNextRunAsync(CancellationToken cancellationToken)
    {
        var lastCompletedAtUtc = await GetLastCompletedAtUtcAsync(cancellationToken);
        if (!lastCompletedAtUtc.HasValue)
        {
            return TimeSpan.Zero;
        }

        var nextRunAtUtc = lastCompletedAtUtc.Value.Add(RetryInterval);
        var delay = nextRunAtUtc - DateTimeOffset.UtcNow;
        return delay > TimeSpan.Zero ? delay : TimeSpan.Zero;
    }

    /// <summary>
    /// 从 FusionCache 读取上次补发完成时间,并与本地 <see cref="_lastCompletedAtUtc"/> 交叉校验后取最新值
    /// 当缓存不可用时回退为本地值
    /// </summary>
    /// <param name="cancellationToken">取消令牌</param>
    /// <returns>上次补发完成时间;若无记录则返回 <see langword="null"/></returns>
    private async Task<DateTimeOffset?> GetLastCompletedAtUtcAsync(
        CancellationToken cancellationToken
    )
    {
        try
        {
            var cache = await fusionCache.TryGetAsync<MessageOutboxRetryState>(
                StateCacheKey,
                token: cancellationToken
            );
            return GetLatestCompletedAtUtc(
                cache.HasValue ? cache.Value.CompletedAtUtc : null,
                _lastCompletedAtUtc
            );
        }
        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
        {
            throw;
        }
        catch (Exception ex)
        {
            logger.LogWarning(ex, "读取Outbox重试缓存状态失败");
            return _lastCompletedAtUtc;
        }
    }

    /// <summary>
    /// 启动时从缓存读取并记录上次补发状态,恢复本地 <see cref="_lastCompletedAtUtc"/>
    /// </summary>
    /// <param name="cancellationToken">取消令牌</param>
    private async Task LogLastRetryStateAsync(CancellationToken cancellationToken)
    {
        try
        {
            var cache = await fusionCache.TryGetAsync<MessageOutboxRetryState>(
                StateCacheKey,
                token: cancellationToken
            );
            if (!cache.HasValue)
            {
                return;
            }

            _lastCompletedAtUtc = cache.Value.CompletedAtUtc;
            logger.LogInformation(
                "Outbox 上次重试完成时间: CompletedAtUtc={CompletedAtUtc}, Publish={PublishCount}, Consume={ConsumeCount}",
                cache.Value.CompletedAtUtc,
                cache.Value.PublishRetryCount,
                cache.Value.ConsumeRetryCount
            );
        }
        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
        {
            throw;
        }
        catch (Exception ex)
        {
            logger.LogWarning(ex, "读取Outbox上次重试状态失败");
        }
    }

    /// <summary>
    /// 从缓存值和本地值中取最新的补发完成时间
    /// </summary>
    /// <param name="cacheCompletedAtUtc">缓存中记录的上次完成时间</param>
    /// <param name="localCompletedAtUtc">本地记录的上次完成时间</param>
    /// <returns>两者中较新的时间;均可为 <see langword="null"/></returns>
    private static DateTimeOffset? GetLatestCompletedAtUtc(
        DateTimeOffset? cacheCompletedAtUtc,
        DateTimeOffset? localCompletedAtUtc
    )
    {
        if (!cacheCompletedAtUtc.HasValue)
        {
            return localCompletedAtUtc;
        }

        if (!localCompletedAtUtc.HasValue)
        {
            return cacheCompletedAtUtc;
        }

        return cacheCompletedAtUtc > localCompletedAtUtc
            ? cacheCompletedAtUtc
            : localCompletedAtUtc;
    }

    /// <summary>
    /// 异步等待指定时长,若延迟 ≤ 0 则立即返回
    /// </summary>
    /// <param name="delay">等待时长</param>
    /// <param name="cancellationToken">取消令牌</param>
    private static async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken)
    {
        if (delay <= TimeSpan.Zero)
        {
            return;
        }

        await Task.Delay(delay, cancellationToken);
    }
}
⚠⚠    以下内容为AI分析的结果,请根据实际情况进行判断。

代码解释

这是一个 Outbox 消息补发后台服务,用于在分布式消息队列系统中周期性地重试发送失败或消费失败的消息。

核心功能

这个后台服务继承自 BackgroundService,主要负责:

  1. 周期性扫描:定期检查需要重试的消息
  2. 分布式协调:使用分布式锁确保多实例环境下只有一个实例执行补发
  3. 状态持久化:通过缓存记录补发状态,实现跨实例状态共享

关键设计要点

1. 构造函数依赖注入

public sealed class MessageOutboxRetryBackgroundService(
    IMessageOutboxRetryService retryService,      // 实际执行补发逻辑
    IFusionCache fusionCache,                      // 分布式缓存,存储补发状态
    IDistributedLockProvider distributedLockProvider, // 分布式锁
    ILogger<...> logger                            // 日志记录
)

2. 配置常量

  • BatchSize = 50:每轮处理50条消息
  • RetryInterval = 1分钟:两轮补发之间的最小间隔
  • StateCacheDuration = 7天:状态在缓存中的保留时间
  • LockWaitTimeout = 3秒:获取锁的超时时间

3. 主执行流程 (ExecuteAsync)

启动 → 记录上次状态 → 进入循环:
    ├─ 计算需要等待的时间
    ├─ 如果需要等待 → 延迟后继续
    ├─ 执行一轮补发 (RunOnceAsync)
    └─ 异常处理 → 等待1分钟后重试

4. 单轮补发流程 (RunOnceAsync)

尝试获取分布式锁
    ├─ 未获取到 → 跳过本轮,更新本地完成时间
    └─ 获取成功 →
        ├─ 再次检查是否需要等待(双重检查)
        ├─ 调用 retryService.RetryAsync() 执行实际补发
        ├─ 保存补发结果到缓存
        └─ 记录日志

5. 状态管理机制

采用双重状态存储策略:

  • 本地状态 (_lastCompletedAtUtc):进程内存中的完成时间
  • 缓存状态 (FusionCache):跨实例共享的完成时间

读取时取两者中较新的值,确保多实例环境下的一致性。

6. 分布式锁的作用

await using var lockHandle = await TryAcquireRetryLockAsync(cancellationToken);
if (lockHandle == null) {
    // 其他实例正在执行,跳过
}

在多实例部署时,确保同一时刻只有一个实例执行补发,避免重复处理。

7. 时间计算逻辑 (GetDelayBeforeNextRunAsync)

上次完成时间 + 1分钟间隔 = 下次执行时间
下次执行时间 - 当前时间 = 需要等待的时长

如果计算结果 ≤ 0,则立即执行。

关键方法说明

方法作用
ExecuteAsync后台服务主循环
RunOnceAsync执行单轮补发(带分布式锁)
SaveRetryStateAsync将补发结果保存到缓存
GetLastCompletedAtUtcAsync从缓存读取上次完成时间(带本地回退)
GetDelayBeforeNextRunAsync计算距下次执行的等待时间
TryAcquireRetryLockAsync尝试获取分布式锁

异常处理

  • 取消操作:检测到 stoppingToken.IsCancellationRequested 时优雅退出
  • 一般异常:记录错误日志后等待1分钟再重试
  • 缓存读取失败:回退使用本地状态,不中断主流程

适用场景

这个服务适用于需要可靠消息投递的场景,特别是:

  • 分布式事务的最终一致性保证
  • 消息队列发送失败的自动重试
  • 消费失败消息的补偿处理
  • 多实例部署的高可用系统
评论加载中...