Root / src / Dpz.Core.Web.Jobs / Hangfire / BackupProgramActivator.cs

using System.Collections.Concurrent;
using System.IO.Compression;
using Dpz.Core.Infrastructure;
using Hangfire;
using Hangfire.Annotations;

namespace Dpz.Core.Web.Jobs.Hangfire;

[UsedImplicitly]
public class BackupProgramActivator(
    IConfiguration configuration,
    ILogger<BackupProgramActivator> logger
) : JobActivator
{
    private readonly CancellationTokenSource _cts = new();
    private int _fileCount;
    private int _folderCount;
    private int _errorCount;
    private long _totalSize;
    private int _processedCount;
    private DateTime _lastLogTime;

    private readonly BackupJobSettings _settings =
        configuration.GetSection("BackupSettings").Get<BackupJobSettings>()
        ?? new BackupJobSettings("", [], [], []);

    /// <summary>
    /// 启动程序目录备份任务。
    /// </summary>
    /// <exception cref="BusinessException">程序备份源目录未配置或不存在。</exception>
    public async Task StartAsync()
    {
        if (
            string.IsNullOrWhiteSpace(_settings.BackupProgramPath)
            || !Directory.Exists(_settings.BackupProgramPath)
        )
        {
            logger.LogWarning(
                "未配置程序备份路径或路径无效: {BackupProgramPath}",
                _settings.BackupProgramPath
            );
            return;
        }

        await MoveAsync(_settings.BackupProgramPath);
    }

    /// <summary>
    /// 将程序目录复制到 rclone 挂载的目标目录。
    /// </summary>
    /// <param name="backupProgramPath">需要备份的程序根目录。</param>
    /// <returns>本次备份生成的目标根目录。</returns>
    /// <exception cref="BusinessException">目标根目录未配置、不存在或没有可备份文件。</exception>
    private async Task MoveAsync(string backupProgramPath)
    {
        var startTime = DateTime.Now;
        ResetStatistics();

        var backupRootPth = configuration.GetValue<string>("BackupRootPath");
        if (string.IsNullOrWhiteSpace(backupRootPth) || !Directory.Exists(backupRootPth))
        {
            logger.LogError(
                "未配置程序备份目标根目录或目录不存在: {BackupProgramRootPath}",
                backupRootPth
            );
            throw new BusinessException("未配置程序备份目标根目录或目录不存在");
        }

        var timestamp = DateTime.Now.ToString("yyyyMMdd");
        var destinationRootPath = IncrementUniqueDestinationRootPath(
            BuildDestinationRootPath(backupRootPth, timestamp)
        );

        logger.LogInformation(
            "开始复制程序备份: {Source} -> {Destination}",
            backupProgramPath,
            destinationRootPath
        );

        var filesToCopy = new ConcurrentBag<FileEntry>();
        var sourceDirectory = new DirectoryInfo(backupProgramPath);
        await CollectFilesAsync(sourceDirectory, sourceDirectory.FullName, filesToCopy);

        var totalFiles = filesToCopy.Count;
        logger.LogInformation(
            "已扫描程序备份文件: {TotalFiles} 个文件, {TotalFolders} 个目录 (已忽略部分文件)",
            totalFiles,
            _folderCount
        );

        if (totalFiles == 0)
        {
            logger.LogWarning("没有找到需要备份的程序文件");
            return;
        }

        Directory.CreateDirectory(destinationRootPath);
        _lastLogTime = DateTime.Now;

        var maxDegreeOfParallelism = Math.Clamp(Environment.ProcessorCount * 2, 4, 64);
        await Parallel.ForEachAsync(
            filesToCopy,
            new ParallelOptions
            {
                MaxDegreeOfParallelism = maxDegreeOfParallelism,
                CancellationToken = _cts.Token,
            },
            async (fileEntry, cancellationToken) =>
            {
                await CopyFileAsync(fileEntry, destinationRootPath, totalFiles, cancellationToken);
            }
        );

        LogCopyCompletionStatistics(destinationRootPath, startTime, maxDegreeOfParallelism);
    }

    /// <summary>
    /// 构建按日期和时间戳分组的程序备份目标目录。
    /// </summary>
    /// <param name="backupProgramRootPath">rclone 挂载的备份根目录。</param>
    /// <param name="timestamp">本次备份时间戳。</param>
    /// <returns>完整的备份目标目录。</returns>
    private static string BuildDestinationRootPath(string backupProgramRootPath, string timestamp)
    {
        var date = DateTime.Now;
        var backupPath = new List<string>
        {
            "program",
            date.Year.ToString(),
            date.Month.ToString(),
            $"backup_{timestamp}",
        };
#if DEBUG
        backupPath.Insert(1, "Test");
#endif
        return Path.Combine(backupProgramRootPath, Path.Combine(backupPath.ToArray()));
    }

    /// <summary>
    /// 确保备份目标目录不会覆盖已有备份。
    /// </summary>
    /// <param name="destinationRootPath">原始目标目录。</param>
    /// <returns>不存在冲突的目标目录。</returns>
    private static string IncrementUniqueDestinationRootPath(string destinationRootPath)
    {
        if (!Directory.Exists(destinationRootPath))
        {
            return destinationRootPath;
        }

        for (var index = 1; ; index++)
        {
            var candidate = $"{destinationRootPath}_{index}";
            if (!Directory.Exists(candidate))
            {
                return candidate;
            }
        }
    }

    /// <summary>
    /// 复制单个文件并更新复制统计信息。
    /// </summary>
    /// <param name="fileEntry">待复制文件信息。</param>
    /// <param name="destinationRootPath">本次备份的目标根目录。</param>
    /// <param name="totalFiles">本次备份需要处理的文件总数。</param>
    /// <param name="cancellationToken">取消令牌。</param>
    private async Task CopyFileAsync(
        FileEntry fileEntry,
        string destinationRootPath,
        int totalFiles,
        CancellationToken cancellationToken
    )
    {
        var destinationPath = Path.Combine(destinationRootPath, fileEntry.RelativePath);
        try
        {
            var destinationDirectory = Path.GetDirectoryName(destinationPath);
            if (!string.IsNullOrWhiteSpace(destinationDirectory))
            {
                Directory.CreateDirectory(destinationDirectory);
            }

            var fileInfo = new FileInfo(fileEntry.FullPath);
            await using var sourceStream = new FileStream(
                fileEntry.FullPath,
                FileMode.Open,
                FileAccess.Read,
                FileShare.ReadWrite | FileShare.Delete,
                1024 * 1024,
                FileOptions.SequentialScan | FileOptions.Asynchronous
            );
            await using var destinationStream = new FileStream(
                destinationPath,
                FileMode.Create,
                FileAccess.Write,
                FileShare.None,
                1024 * 1024,
                FileOptions.SequentialScan | FileOptions.Asynchronous
            );

            await sourceStream.CopyToAsync(destinationStream, 1024 * 1024, cancellationToken);

            try
            {
                File.SetLastWriteTimeUtc(destinationPath, fileInfo.LastWriteTimeUtc);
            }
            catch (Exception ex)
            {
                logger.LogInformation(ex, "无法设置备份文件修改时间: {Path}", destinationPath);
            }

            var copiedSize = Interlocked.Add(ref _totalSize, fileInfo.Length);
            var copiedFiles = Interlocked.Increment(ref _fileCount);
            var processedCount = Interlocked.Increment(ref _processedCount);
            LogCopyProgressIfNeeded(processedCount, totalFiles, copiedFiles, copiedSize);
        }
        catch (Exception ex)
        {
            Interlocked.Increment(ref _errorCount);
            var processedCount = Interlocked.Increment(ref _processedCount);
            logger.LogError(
                ex,
                "复制程序备份文件失败: {Source} -> {Destination}",
                fileEntry.FullPath,
                destinationPath
            );
            LogCopyProgressIfNeeded(processedCount, totalFiles, _fileCount, _totalSize);
        }
    }

    /// <summary>
    /// 按时间或文件数量间隔记录目录复制进度。
    /// </summary>
    /// <param name="currentProcessed">已处理文件数。</param>
    /// <param name="totalFiles">总文件数。</param>
    /// <param name="copiedFiles">已成功复制文件数。</param>
    /// <param name="copiedSize">已成功复制的字节数。</param>
    private void LogCopyProgressIfNeeded(
        int currentProcessed,
        int totalFiles,
        int copiedFiles,
        long copiedSize
    )
    {
        if (currentProcessed % 2000 != 0 && (DateTime.Now - _lastLogTime).TotalSeconds < 5)
        {
            return;
        }

        _lastLogTime = DateTime.Now;
        var progress = totalFiles > 0 ? (currentProcessed * 100.0) / totalFiles : 0;
        var sizeGb = copiedSize / (1024.0 * 1024.0 * 1024.0);
        logger.LogInformation(
            "程序备份复制进度: {Progress:F2}% ({Processed}/{Total}), 已复制文件: {CopiedFiles}, 已复制大小: {Size:F2} GB, 错误数: {ErrorCount}",
            progress,
            currentProcessed,
            totalFiles,
            copiedFiles,
            sizeGb,
            _errorCount
        );
    }

    /// <summary>
    /// 记录目录复制完成后的统计信息。
    /// </summary>
    /// <param name="destinationRootPath">本次备份的目标根目录。</param>
    /// <param name="startTime">备份开始时间。</param>
    /// <param name="maxDegreeOfParallelism">复制阶段使用的最大并发数。</param>
    private void LogCopyCompletionStatistics(
        string destinationRootPath,
        DateTime startTime,
        int maxDegreeOfParallelism
    )
    {
        var elapsed = DateTime.Now - startTime;
        var finalSizeMb = _totalSize / (1024.0 * 1024.0);
        var finalSizeGb = _totalSize / (1024.0 * 1024.0 * 1024.0);
        var message =
            "程序备份复制完成: {Destination}\n"
            + "统计信息:\n"
            + "  - 文件数: {FileCount}\n"
            + "  - 目录数: {FolderCount}\n"
            + "  - 错误数: {ErrorCount}\n"
            + "  - 总大小: {TotalSize:F2} GB ({TotalSizeMB:F2} MB)\n"
            + "  - 并发数: {MaxDegreeOfParallelism}\n"
            + "  - 耗时: {Elapsed}";

        if (_errorCount > 0)
        {
            logger.LogWarning(
                message,
                destinationRootPath,
                _fileCount,
                _folderCount,
                _errorCount,
                finalSizeGb,
                finalSizeMb,
                maxDegreeOfParallelism,
                elapsed
            );
            return;
        }

        logger.LogInformation(
            message,
            destinationRootPath,
            _fileCount,
            _folderCount,
            _errorCount,
            finalSizeGb,
            finalSizeMb,
            maxDegreeOfParallelism,
            elapsed
        );
    }

    /// <summary>
    /// 重置本次备份任务的统计数据。
    /// </summary>
    private void ResetStatistics()
    {
        _fileCount = 0;
        _folderCount = 0;
        _errorCount = 0;
        _totalSize = 0;
        _processedCount = 0;
        _lastLogTime = DateTime.Now;
    }

    /// <summary>
    /// 递归扫描目录并收集未被忽略的文件。
    /// </summary>
    /// <param name="directory">当前扫描目录。</param>
    /// <param name="rootPath">备份源根目录。</param>
    /// <param name="files">用于保存扫描结果的集合。</param>
    private async Task CollectFilesAsync(
        DirectoryInfo directory,
        string rootPath,
        ConcurrentBag<FileEntry> files
    )
    {
        try
        {
            Interlocked.Increment(ref _folderCount);

            // 收集当前目录的所有文件
            var directoryFiles = directory.GetFiles();
            foreach (var file in directoryFiles)
            {
                if (ShouldIgnore(file, rootPath))
                {
                    continue;
                }

                try
                {
                    var relativePath = Path.GetRelativePath(rootPath, file.FullName);
                    files.Add(new FileEntry(file.FullName, relativePath));
                }
                catch (Exception ex)
                {
                    Interlocked.Increment(ref _errorCount);
                    logger.LogError(ex, "收集文件信息失败 {FilePath}，已跳过", file.FullName);
                }
            }

            // 并行处理子目录
            var subDirectories = directory.GetDirectories();
            await Parallel.ForEachAsync(
                subDirectories,
                new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount },
                async (subDirectory, _) =>
                {
                    if (ShouldIgnoreDirectory(subDirectory, rootPath))
                    {
                        return;
                    }
                    await ParallelFolderAsync(rootPath, files, subDirectory);
                }
            );
        }
        catch (UnauthorizedAccessException ex)
        {
            Interlocked.Increment(ref _errorCount);
            logger.LogError(
                ex,
                "没有访问目录 {DirectoryPath} 的权限，已跳过该目录",
                directory.FullName
            );
        }
        catch (Exception ex)
        {
            Interlocked.Increment(ref _errorCount);
            logger.LogError(
                ex,
                "处理目录 {DirectoryPath} 时发生错误，已跳过该目录",
                directory.FullName
            );
        }
    }

    /// <summary>
    /// 验证忽略路径是否合法（必须在备份根目录下）
    /// </summary>
    /// <param name="ignorePath">配置中的忽略路径。</param>
    /// <param name="rootPath">备份源根目录。</param>
    /// <param name="normalizedIgnorePath">规范化后的忽略路径。</param>
    /// <returns>忽略路径是否合法。</returns>
    private bool IsValidIgnorePath(
        string ignorePath,
        string rootPath,
        out string normalizedIgnorePath
    )
    {
        normalizedIgnorePath = string.Empty;

        try
        {
            // 规范化路径，处理相对路径和 .. 等情况
            var fullIgnorePath = Path.GetFullPath(ignorePath);
            var fullRootPath = Path.GetFullPath(rootPath);

            // 确保两个路径都以目录分隔符结尾，便于比较
            if (!fullRootPath.EndsWith(Path.DirectorySeparatorChar))
            {
                fullRootPath += Path.DirectorySeparatorChar;
            }

            if (!fullIgnorePath.EndsWith(Path.DirectorySeparatorChar))
            {
                fullIgnorePath += Path.DirectorySeparatorChar;
            }

            // 检查忽略路径是否在备份根路径下
            if (!fullIgnorePath.StartsWith(fullRootPath, StringComparison.OrdinalIgnoreCase))
            {
                // logger.LogWarning(
                //     "忽略路径配置不合理: {IgnorePath} 不在备份路径 {RootPath} 下，已跳过此配置",
                //     ignorePath,
                //     rootPath
                // );
                return false;
            }

            normalizedIgnorePath = fullIgnorePath.TrimEnd(Path.DirectorySeparatorChar);
            return true;
        }
        catch (Exception ex)
        {
            logger.LogWarning(ex, "忽略路径配置无效: {IgnorePath}，已跳过此配置", ignorePath);
            return false;
        }
    }

    /// <summary>
    /// 判断文件是否匹配忽略配置。
    /// </summary>
    /// <param name="file">待判断文件。</param>
    /// <param name="rootPath">备份源根目录。</param>
    /// <returns>是否应该跳过该文件。</returns>
    private bool ShouldIgnore(FileInfo file, string rootPath)
    {
        // 检查文件名 -> 大小写敏感
        if (_settings.IgnoreFiles.Any(x => x == file.Name))
        {
            return true;
        }

        // 检查扩展名 -> 大小写敏感
        if (_settings.IgnoreExtensions.Any(x => x == file.Extension))
        {
            return true;
        }

        // 检查文件所在目录是否应该被忽略
        if (file.Directory != null && ShouldIgnoreDirectory(file.Directory, rootPath))
        {
            return true;
        }

        return false;
    }

    /// <summary>
    /// 判断目录是否匹配忽略路径配置。
    /// </summary>
    /// <param name="dir">待判断目录。</param>
    /// <param name="rootPath">备份源根目录。</param>
    /// <returns>是否应该跳过该目录。</returns>
    private bool ShouldIgnoreDirectory(DirectoryInfo dir, string rootPath)
    {
        if (_settings.IgnorePaths.Length == 0)
        {
            return false;
        }

        var fullDirPath = Path.GetFullPath(dir.FullName);

        foreach (var ignorePath in _settings.IgnorePaths)
        {
            if (string.IsNullOrWhiteSpace(ignorePath))
            {
                continue;
            }

            if (!IsValidIgnorePath(ignorePath, rootPath, out var normalizedIgnorePath))
            {
                continue;
            }

            // 检查目录是否完全匹配忽略路径
            if (fullDirPath.Equals(normalizedIgnorePath, StringComparison.OrdinalIgnoreCase))
            {
                return true;
            }

            // 检查目录是否在忽略路径下
            if (
                fullDirPath.StartsWith(
                    normalizedIgnorePath + Path.DirectorySeparatorChar,
                    StringComparison.OrdinalIgnoreCase
                )
            )
            {
                return true;
            }
        }

        return false;
    }

    /// <summary>
    /// 并行扫描子目录并收集备份文件。
    /// </summary>
    /// <param name="rootPath">备份源根目录。</param>
    /// <param name="files">用于保存扫描结果的集合。</param>
    /// <param name="subDirectory">需要扫描的子目录。</param>
    private async Task ParallelFolderAsync(
        string rootPath,
        ConcurrentBag<FileEntry> files,
        DirectoryInfo subDirectory
    )
    {
        try
        {
            await CollectFilesAsync(subDirectory, rootPath, files);
        }
        catch (Exception ex)
        {
            Interlocked.Increment(ref _errorCount);
            logger.LogError(ex, "处理子目录异常: {Path}", subDirectory.FullName);
        }
    }

    private readonly record struct FileEntry(string FullPath, string RelativePath);

    private record BackupJobSettings(
        string? BackupProgramPath,
        string[] IgnorePaths,
        string[] IgnoreFiles,
        string[] IgnoreExtensions,
        CompressionLevel CompressionLevel = CompressionLevel.Fastest
    );
}

⚠⚠ 以下内容为AI分析的结果，请根据实际情况进行判断。

代码解释

这是一个用于程序文件备份的 Hangfire 后台任务类，主要功能是将指定目录的文件复制到远程备份位置（通过 rclone 挂载）。

核心功能

1. 类的作用

继承自 JobActivator，作为 Hangfire 的任务激活器
实现程序文件的并行备份，支持忽略规则、进度跟踪和错误处理

2. 主要组件

配置和状态

private readonly BackupJobSettings _settings  // 备份配置（忽略规则等）
private int _fileCount, _folderCount, _errorCount  // 统计计数器
private long _totalSize  // 已备份文件总大小
private int _processedCount  // 已处理文件数

备份设置（BackupJobSettings）

BackupProgramPath: 源目录路径
IgnorePaths/IgnoreFiles/IgnoreExtensions: 忽略规则
CompressionLevel: 压缩级别（虽然定义了但代码中未使用）

3. 工作流程

启动入口 - StartAsync()

检查源目录配置 → 调用 MoveAsync() 执行备份

核心备份逻辑 - MoveAsync()

验证目标根目录（从配置读取 BackupRootPath）

生成唯一目标路径

格式: backup_root/program/年/月/backup_yyyyMMdd[_序号]
DEBUG模式: backup_root/program/Test/年/月/...

扫描源文件 - 递归收集需要备份的文件
并行复制 - 使用 Parallel.ForEachAsync 高效复制
记录统计 - 输出完成信息

4. 关键特性

智能忽略机制

路径验证: IsValidIgnorePath() 确保忽略路径在备份根目录下
文件忽略: 匹配文件名/扩展名（大小写敏感）
目录忽略: 匹配完整路径或路径前缀

并行处理优化

// 文件扫描阶段 - 使用 CPU 核心数
MaxDegreeOfParallelism = Environment.ProcessorCount

// 文件复制阶段 - CPU核心数 * 2（限制4-64）
MaxDegreeOfParallelism = Math.Clamp(Environment.ProcessorCount * 2, 4, 64)

文件复制优化

// 1MB 缓冲区，顺序扫描，异步 I/O
FileOptions.SequentialScan | FileOptions.Asynchronous
FileShare.ReadWrite | FileShare.Delete  // 允许源文件被其他进程使用

进度监控

每复制 2000 个文件或每 5 秒记录一次进度
显示百分比、文件数、已复制大小和错误数

5. 容错机制

线程安全计数: 使用 Interlocked 操作
异常捕获:
- 文件复制失败不中断整体流程
- 目录访问权限异常单独处理
- 路径验证失败跳过该配置
取消令牌: 支持通过 CancellationTokenSource 取消任务

6. 日志输出示例

开始复制程序备份: /app -> /backup/program/2024/1/backup_20240115
已扫描程序备份文件: 5000 个文件, 320 个目录
程序备份复制进度: 45.20% (2260/5000), 已复制文件: 2255, 已复制大小: 3.45 GB, 错误数: 5
程序备份复制完成: /backup/program/2024/1/backup_20240115
  - 文件数: 4995
  - 错误数: 5
  - 总大小: 7.68 GB
  - 并发数: 16
  - 耗时: 00:05:32

7. 使用场景

适用于需要定期备份应用程序目录的场景，特别是：

多文件小文件场景（利用并行优势）
需要精细控制忽略规则
备份到网络存储（如 rclone 挂载的云存储）

8. 潜在问题

⚠️ 未实现压缩功能（CompressionLevel 未使用）
⚠️ 未验证目标路径空间是否足够
⚠️ 大文件场景下的内存占用较高（1MB 缓冲区）

评论加载中...