Root / src / Dpz.Core.Service.Mediator / Features / Code / Commands / SyncCodeFileSystemEntriesRequest.cs

using System.Collections.Concurrent;
using Dpz.Core.Service.Mediator.Features.Code.Contracts;

namespace Dpz.Core.Service.Mediator.Features.Code.Commands;

/// <summary>
/// 同步代码文件系统节点到数据库的请求。
/// </summary>
public class SyncCodeFileSystemEntriesRequest : IRequest<CodeFileSystemSyncSummary> { }

/// <summary>
/// 同步代码文件系统节点到数据库。
/// </summary>
public class SyncCodeFileSystemEntriesHandler(
    IRepository<CodeFileSystemEntry> repository,
    IConfiguration configuration,
    IFusionCache fusionCache,
    IMediator mediator,
    ILogger<SyncCodeFileSystemEntriesHandler> logger
) : IRequestHandler<SyncCodeFileSystemEntriesRequest, CodeFileSystemSyncSummary>
{
    private const string CachePrefix =
        "Dpz.Core.Service.RepositoryServiceImpl.CodeFileSystemEntryService";

    /// <summary>
    /// 同步本地代码文件系统节点到数据库。
    /// </summary>
    public async ValueTask<CodeFileSystemSyncSummary> Handle(
        SyncCodeFileSystemEntriesRequest request,
        CancellationToken cancellationToken
    )
    {
        var syncStopwatch = Stopwatch.StartNew();
        var codeView =
            configuration.GetSection("CodeView").Get<CodeViewOption>()
            ?? throw new InvalidConfigurationException("CodeView 配置无效");
        var sourceRoot = codeView.SourceCodeRoot;

        if (
            string.IsNullOrEmpty(codeView.SourceCodeRoot)
            || !Directory.Exists(codeView.SourceCodeRoot)
        )
        {
            throw new InvalidConfigurationException("CodeView:SourceCodeRoot 配置无效");
        }

        logger.LogInformation("开始同步代码，源路径: {SourceRoot}", sourceRoot);

        var fileSystemInfos = GetAllFileSystemInfos(sourceRoot, codeView);
        var fileSystemMap = new Dictionary<string, FileSystemInfo>(
            StringComparer.OrdinalIgnoreCase
        );

        foreach (var item in fileSystemInfos)
        {
            var relativeSegments = GetRelativeSegments(sourceRoot, item.FullName);
            if (relativeSegments.Count == 0)
            {
                continue;
            }

            var key = BuildPathKey(relativeSegments);
            fileSystemMap[key] = item;
        }

        logger.LogInformation("扫描完成，共 {Count} 个文件系统项", fileSystemMap.Count);

        var dbEntries = await repository.SearchFor(x => true).ToListAsync(cancellationToken);
        var dbMap = dbEntries.ToDictionary(
            x => BuildPathKey(x.PathSegments),
            StringComparer.OrdinalIgnoreCase
        );

        var insertList = new ConcurrentBag<CodeFileSystemEntry>();
        var updateList = new ConcurrentBag<CodeFileSystemEntry>();
        var deleteIds = new List<ObjectId>();

        var maxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2);
        await Parallel.ForEachAsync(
            fileSystemMap,
            new ParallelOptions
            {
                MaxDegreeOfParallelism = maxDegreeOfParallelism,
                CancellationToken = cancellationToken,
            },
            async (pair, ct) =>
            {
                if (!dbMap.TryGetValue(pair.Key, out var exist))
                {
                    var entry = await BuildEntryAsync(codeView, pair.Value, ct);
                    if (entry != null)
                    {
                        insertList.Add(entry);
                    }

                    return;
                }

                var updated = await BuildUpdatedEntryAsync(codeView, exist, pair.Value, ct);
                if (updated != null)
                {
                    updateList.Add(updated);
                }
            }
        );

        foreach (var entry in dbEntries)
        {
            var key = BuildPathKey(entry.PathSegments);
            if (!fileSystemMap.ContainsKey(key))
            {
                deleteIds.Add(entry.Id);
            }
        }

        var insertListFinal = insertList.ToList();
        var updateListFinal = updateList.ToList();

        if (insertListFinal.Count > 0)
        {
            await repository.InsertAsync(insertListFinal, cancellationToken);
            logger.LogInformation("已新增 {Count} 条记录", insertListFinal.Count);
        }

        if (updateListFinal.Count > 0)
        {
            await repository.UpdateAsync(updateListFinal, cancellationToken);
            logger.LogInformation("已更新 {Count} 条记录", updateListFinal.Count);
        }

        if (deleteIds.Count > 0)
        {
            var filter = Builders<CodeFileSystemEntry>.Filter.In(x => x.Id, deleteIds);
            await repository.DeleteAsync(filter, cancellationToken);
            logger.LogInformation("已删除 {Count} 条记录", deleteIds.Count);
        }

        syncStopwatch.Stop();
        var summary = new CodeFileSystemSyncSummary
        {
            InsertCount = insertListFinal.Count,
            UpdateCount = updateListFinal.Count,
            DeleteCount = deleteIds.Count,
            ElapsedMilliseconds = syncStopwatch.ElapsedMilliseconds,
        };

        logger.LogInformation(
            "代码同步完成，新增:{InsertCount} 条，更新:{UpdateCount} 条，删除:{DeleteCount} 条，耗时:{ElapsedMs}ms",
            summary.InsertCount,
            summary.UpdateCount,
            summary.DeleteCount,
            summary.ElapsedMilliseconds
        );

        if (insertListFinal.Count > 0)
        {
            await mediator.Send(
                new CodeCompatibleRequest
                {
                    NewEntries = insertListFinal
                        .Select(x => new CodeCompatibleEntry
                        {
                            ParentPathSegments = x.ParentPathSegments,
                            Name = x.Name,
                        })
                        .ToList(),
                },
                cancellationToken
            );
        }

        if (summary.UpdateCount > 0 || summary.DeleteCount > 0 || summary.InsertCount > 0)
        {
            await fusionCache.RemoveByTagAsync(
                [
                    GetMethodTag("FindByPathAsync"),
                    GetMethodTag("GetChildrenAsync"),
                    GetMethodTag("SearchAsync"),
                ],
                token: cancellationToken
            );
        }

        return summary;
    }

    private List<FileSystemInfo> GetAllFileSystemInfos(string path, CodeViewOption codeView)
    {
        if (string.IsNullOrEmpty(path))
        {
            return [];
        }

        var dir = new DirectoryInfo(path);
        if (!dir.Exists)
        {
            return [];
        }

        var allFiles = new ConcurrentBag<FileSystemInfo>();

        try
        {
            var fileSystemInfos = dir.GetFileSystemInfos();
            var subDirectories = new List<DirectoryInfo>();
            var currentFiles = new List<FileSystemInfo>();

            foreach (var item in fileSystemInfos)
            {
                if (IsFiltered(codeView, item.Name))
                {
                    continue;
                }

                if (item is DirectoryInfo subDir)
                {
                    currentFiles.Add(subDir);
                    subDirectories.Add(subDir);
                }
                else
                {
                    currentFiles.Add(item);
                }
            }

            foreach (var file in currentFiles)
            {
                allFiles.Add(file);
            }

            if (subDirectories.Count > 0)
            {
                Parallel.ForEach(
                    subDirectories,
                    new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount / 2 },
                    subDir =>
                    {
                        var subFiles = GetAllFileSystemInfos(subDir.FullName, codeView);
                        foreach (var subFile in subFiles)
                        {
                            allFiles.Add(subFile);
                        }
                    }
                );
            }
        }
        catch (UnauthorizedAccessException ex)
        {
            logger.LogWarning(ex, "无权限访问目录: {Directory}", dir.FullName);
        }

        return allFiles.ToList();
    }

    private async Task<CodeFileSystemEntry?> BuildEntryAsync(
        CodeViewOption codeView,
        FileSystemInfo info,
        CancellationToken cancellationToken
    )
    {
        if (IsFiltered(codeView, info.Name))
        {
            return null;
        }

        var relativeSegments = GetRelativeSegments(codeView.SourceCodeRoot, info.FullName);
        if (relativeSegments.Count == 0)
        {
            return null;
        }

        var parentSegments = relativeSegments.Take(relativeSegments.Count - 1).ToList();

        if (info is DirectoryInfo directoryInfo)
        {
            return new CodeFileSystemEntry
            {
                PathSegments = relativeSegments,
                ParentPathSegments = parentSegments,
                Name = directoryInfo.Name,
                IsDirectory = true,
                Extension = null,
                Size = null,
                Hash = null,
                CodeFileContentType = CodeFileContentType.Unknown,
                FileContent = null,
                CreatedTime = directoryInfo.CreationTime,
                LastWriteTime = directoryInfo.LastWriteTime,
                LastUpdateTime = DateTime.Now,
            };
        }

        if (info is FileInfo fileInfo)
        {
            var extension = GetFileExtension(fileInfo);
            var hash = await ComputeHashAsync(fileInfo, cancellationToken);
            var shouldPreview = ResolveCodeLanguage(codeView, extension, fileInfo.Name) != null;
            var content = shouldPreview
                ? await ReadTextContentAsync(fileInfo, cancellationToken)
                : null;

            return new CodeFileSystemEntry
            {
                PathSegments = relativeSegments,
                ParentPathSegments = parentSegments,
                Name = fileInfo.Name,
                IsDirectory = false,
                Extension = extension,
                Size = fileInfo.Length,
                Hash = hash,
                CodeFileContentType = shouldPreview
                    ? CodeFileContentType.Text
                    : CodeFileContentType.Unknown,
                FileContent = content,
                CreatedTime = fileInfo.CreationTime,
                LastWriteTime = fileInfo.LastWriteTime,
                LastUpdateTime = DateTime.Now,
            };
        }

        return null;
    }

    private async Task<CodeFileSystemEntry?> BuildUpdatedEntryAsync(
        CodeViewOption codeView,
        CodeFileSystemEntry exist,
        FileSystemInfo info,
        CancellationToken cancellationToken
    )
    {
        if (IsFiltered(codeView, info.Name))
        {
            return null;
        }

        if (info is DirectoryInfo directoryInfo)
        {
            if (
                exist.IsDirectory
                && DateTimeAreEqual(exist.LastWriteTime, directoryInfo.LastWriteTime)
                && exist.Name == directoryInfo.Name
            )
            {
                return null;
            }

            exist.IsDirectory = true;
            exist.Name = directoryInfo.Name;
            exist.Extension = null;
            exist.Size = null;
            exist.Hash = null;
            exist.CodeFileContentType = CodeFileContentType.Unknown;
            exist.FileContent = null;
            exist.LastWriteTime = directoryInfo.LastWriteTime;
            exist.LastUpdateTime = DateTime.Now;
            return exist;
        }

        if (info is FileInfo fileInfo)
        {
            var extension = GetFileExtension(fileInfo);
            var hash = await ComputeHashAsync(fileInfo, cancellationToken);
            var shouldPreview = ResolveCodeLanguage(codeView, extension, fileInfo.Name) != null;

            var typeChanged = exist.IsDirectory;
            var hashChanged = !string.Equals(exist.Hash, hash, StringComparison.OrdinalIgnoreCase);
            var sizeChanged = exist.Size != fileInfo.Length;
            var lastWriteTimeChanged = !DateTimeAreEqual(
                exist.LastWriteTime,
                fileInfo.LastWriteTime
            );
            var extensionChanged = !string.Equals(
                exist.Extension,
                extension,
                StringComparison.OrdinalIgnoreCase
            );
            var previewEnabled =
                shouldPreview && exist.CodeFileContentType != CodeFileContentType.Text;
            var previewDisabled =
                !shouldPreview && exist.CodeFileContentType == CodeFileContentType.Text;
            var previewContentMissing = shouldPreview && exist.FileContent == null;

            var needUpdate =
                typeChanged
                || hashChanged
                || sizeChanged
                || lastWriteTimeChanged
                || extensionChanged
                || previewEnabled
                || previewDisabled
                || previewContentMissing;

            if (!needUpdate)
            {
                return null;
            }

            exist.IsDirectory = false;
            exist.Extension = extension;
            exist.Size = fileInfo.Length;
            exist.Hash = hash;
            exist.LastWriteTime = fileInfo.LastWriteTime;
            exist.LastUpdateTime = DateTime.Now;
            exist.CodeFileContentType = shouldPreview
                ? CodeFileContentType.Text
                : CodeFileContentType.Unknown;
            exist.FileContent = shouldPreview
                ? await ReadTextContentAsync(fileInfo, cancellationToken)
                : null;

            if (
                !string.IsNullOrWhiteSpace(exist.AiAnalyzeResult)
                && !string.IsNullOrWhiteSpace(exist.FileContent)
            )
            {
                var lineCount = exist
                    .FileContent.Split(["\r\n", "\r", "\n"], StringSplitOptions.None)
                    .Length;
                if (lineCount < 50)
                {
                    exist.AiAnalyzeResult = null;
                    exist.AiAnalyzeHash = null;
                    exist.AiAnalyzeTime = null;
                }
            }

            return exist;
        }

        return null;
    }

    private async Task<string> ComputeHashAsync(
        FileInfo fileInfo,
        CancellationToken cancellationToken
    )
    {
        try
        {
            await using var stream = fileInfo.OpenRead();
            var bytes = await MD5.HashDataAsync(stream, cancellationToken);
            var builder = new StringBuilder(bytes.Length * 2);
            foreach (var item in bytes)
            {
                builder.Append(item.ToString("x2"));
            }

            return builder.ToString();
        }
        catch (Exception ex)
        {
            logger.LogWarning(ex, "读取文件哈希失败: {FilePath}", fileInfo.FullName);
            return string.Empty;
        }
    }

    private async Task<string?> ReadTextContentAsync(
        FileInfo fileInfo,
        CancellationToken cancellationToken
    )
    {
        try
        {
            using var reader = fileInfo.OpenText();
            return await reader.ReadToEndAsync(cancellationToken);
        }
        catch (Exception ex)
        {
            logger.LogWarning(ex, "读取文件内容失败: {FilePath}", fileInfo.FullName);
            return null;
        }
    }

    private static string? ResolveCodeLanguage(
        CodeViewOption codeView,
        string? extension,
        string? name
    )
    {
        foreach (var key in GetLanguageLookupKeys(extension, name))
        {
            if (
                TryGetConfiguredLanguage(
                    codeView.ExtensionToLanguage,
                    key,
                    out var configuredLanguage
                )
            )
            {
                return configuredLanguage;
            }
        }

        return null;
    }

    private static bool TryGetConfiguredLanguage(
        IReadOnlyDictionary<string, string>? extensionToLanguage,
        string key,
        out string? language
    )
    {
        language = null;
        if (extensionToLanguage == null)
        {
            return false;
        }

        if (extensionToLanguage.TryGetValue(key, out var exactLanguage))
        {
            language = exactLanguage;
            return true;
        }

        var matched = extensionToLanguage.FirstOrDefault(x =>
            string.Equals(x.Key, key, StringComparison.OrdinalIgnoreCase)
        );
        if (matched.Key == null)
        {
            return false;
        }

        language = matched.Value;
        return true;
    }

    private static IEnumerable<string> GetLanguageLookupKeys(string? extension, string? name)
    {
        var primary = string.IsNullOrWhiteSpace(extension) ? name : extension;
        foreach (var key in ExpandLanguageLookupKey(primary))
        {
            yield return key;
        }

        if (
            !string.IsNullOrWhiteSpace(name)
            && !string.Equals(name, primary, StringComparison.OrdinalIgnoreCase)
        )
        {
            foreach (var key in ExpandLanguageLookupKey(name))
            {
                yield return key;
            }
        }
    }

    private static IEnumerable<string> ExpandLanguageLookupKey(string? value)
    {
        if (string.IsNullOrWhiteSpace(value))
        {
            yield break;
        }

        var normalized = value.Trim().ToLowerInvariant();
        yield return normalized;

        if (normalized.StartsWith('.'))
        {
            yield return normalized.TrimStart('.');
        }
        else
        {
            yield return "." + normalized;
        }
    }

    private static bool DateTimeAreEqual(DateTime dt1, DateTime dt2)
    {
        var diffSeconds = (int)(dt1 - dt2).TotalSeconds;
        return diffSeconds == 0;
    }

    private static List<string> GetRelativeSegments(string root, string fullPath)
    {
        var relativePath = Path.GetRelativePath(root, fullPath);
        if (relativePath == ".")
        {
            return [];
        }

        return relativePath
            .Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)
            .Where(x => !string.IsNullOrWhiteSpace(x))
            .ToList();
    }

    private static bool IsFiltered(CodeViewOption codeView, string name)
    {
        return codeView.Filters?.Any(x => x.WildCardMatch(name)) == true;
    }

    private static string GetFileExtension(FileInfo fileInfo)
    {
        var extension = fileInfo.Extension;
        return (string.IsNullOrEmpty(extension) ? fileInfo.Name : extension).ToLower();
    }

    private static string BuildPathKey(IEnumerable<string> segments)
    {
        return string.Join("/", segments);
    }

    private static string GetMethodTag(string methodName)
    {
        return CachePrefix + ":" + methodName;
    }
}

⚠⚠ 以下内容为AI分析的结果，请根据实际情况进行判断。

代码解释

这是一个基于 MediatR 模式的代码文件系统同步处理器，主要功能是将本地代码文件系统的结构同步到数据库中。

核心功能

1. 主要类型

SyncCodeFileSystemEntriesRequest: 同步请求类（空请求体）
SyncCodeFileSystemEntriesHandler: 同步处理器（实现 IRequestHandler 接口）

2. 同步流程 (Handle 方法)

1. 读取配置 → 2. 扫描文件系统 → 3. 对比数据库 → 4. 执行增删改 → 5. 清理缓存

详细步骤：

① 配置验证

// 获取并验证源代码根目录配置
var codeView = configuration.GetSection("CodeView").Get<CodeViewOption>();
var sourceRoot = codeView.SourceCodeRoot;

② 文件系统扫描

// 递归获取所有文件和目录，并构建路径映射
var fileSystemInfos = GetAllFileSystemInfos(sourceRoot, codeView);
var fileSystemMap = new Dictionary<string, FileSystemInfo>();

使用相对路径作为 Key（如 "src/services/UserService.cs"）
支持并行扫描子目录（利用多核性能）
根据配置过滤特定文件/文件夹

③ 数据库对比

// 并行处理每个文件系统项
await Parallel.ForEachAsync(fileSystemMap, async (pair, ct) => {
    if (!dbMap.TryGetValue(pair.Key, out var exist)) {
        // 文件系统存在，数据库不存在 → 新增
        insertList.Add(entry);
    } else {
        // 都存在 → 检查是否需要更新
        updateList.Add(updated);
    }
});

// 查找需要删除的记录（数据库存在但文件系统不存在）
foreach (var entry in dbEntries) {
    if (!fileSystemMap.ContainsKey(key)) {
        deleteIds.Add(entry.Id);
    }
}

④ 批量数据库操作

await repository.InsertAsync(insertListFinal, cancellationToken);
await repository.UpdateAsync(updateListFinal, cancellationToken);
await repository.DeleteAsync(filter, cancellationToken);

⑤ 后续处理

发送兼容性检查请求（针对新增文件）
清理相关缓存标签
返回同步摘要统计

3. 关键辅助方法

`GetAllFileSystemInfos` - 递归扫描文件系统

// 递归获取目录下所有文件和子目录
// 支持并行处理子目录
// 处理访问权限异常

`BuildEntryAsync` - 构建新条目

// 目录: 只存储基本信息
// 文件: 
//   - 计算 MD5 哈希
//   - 判断是否可预览（根据扩展名配置）
//   - 可预览的读取文本内容

`BuildUpdatedEntryAsync` - 构建更新条目

// 检查多个维度判断是否需要更新：
// - 类型变化（文件/目录互转）
// - 哈希值变化
// - 文件大小变化
// - 最后修改时间变化
// - 扩展名变化
// - 预览状态变化
// - 内容缺失

// 特殊处理：如果文件内容少于 50 行，清除 AI 分析结果

`ComputeHashAsync` - 计算文件哈希

// 使用 MD5 算法计算文件哈希值
// 用于检测文件内容是否变化

`ResolveCodeLanguage` - 解析代码语言

// 根据文件扩展名或文件名判断编程语言
// 支持配置映射（如 ".cs" → "csharp"）
// 决定文件是否支持文本预览

4. 性能优化设计

并行处理: 使用 Parallel.ForEachAsync 和 Parallel.ForEach
并发集合: 使用 ConcurrentBag 在多线程环境中安全添加元素
批量操作: 收集完所有变更后批量执行数据库操作
智能缓存: 只在有变更时才清理缓存

5. 数据模型

CodeFileSystemEntry 包含字段：

PathSegments: 路径分段（如 ["src", "services", "UserService.cs"]）
ParentPathSegments: 父路径分段
IsDirectory: 是否为目录
Hash: 文件 MD5 哈希
FileContent: 文本内容（可预览文件）
CodeFileContentType: 内容类型（Text/Unknown）
AiAnalyzeResult: AI 分析结果（会在文件内容少时清除）

6. 返回结果

CodeFileSystemSyncSummary {
    InsertCount,      // 新增数量
    UpdateCount,      // 更新数量
    DeleteCount,      // 删除数量
    ElapsedMilliseconds // 耗时（毫秒）
}

使用场景

适用于需要将代码仓库结构镜像到数据库的场景，如：

代码浏览系统
代码搜索引擎
代码分析平台
文档生成系统

评论加载中...