using System.Diagnostics;
using System.Net;
using System.Runtime.ExceptionServices;
using System.Security.Cryptography;
using Amazon.S3;
using Amazon.S3.Model;
using Dpz.Core.Entity.Base.PublicStruct;
using Dpz.Core.Infrastructure;
using Dpz.Core.Shard.Service;
using Microsoft.Extensions.Logging;

namespace Dpz.Core.Service.ObjectStorage.Services.Impl;

/// <summary>
/// 基于腾讯云 COS S3 兼容 API 的对象存储操作
/// </summary>
public class S3ObjectStorageService(
    IAmazonS3 amazonS3,
    S3ObjectStorageOptions options,
    ICloudInfiniteService cloudInfiniteService,
    ILogger<S3ObjectStorageService> logger
) : IObjectStorageOperation
{
    /// <inheritdoc />
    public string Bucket => options.Bucket ?? throw new BusinessException("S3 bucket is null");

    /// <inheritdoc />
    public async Task<IFileMetadata> UploadAsync(
        Stream stream,
        ICollection<string> path,
        string filename,
        CancellationToken cancellationToken = default
    )
    {
        ArgumentNullException.ThrowIfNull(stream);

        var key = S3ObjectKeyHelper.BuildKey(path, filename);
        var size = stream.CanSeek ? stream.Length : 0;
        await UploadStreamAsync(stream, Bucket, key, cancellationToken);
        var url = S3ObjectKeyHelper.BuildAccessUrl(options.CdnHost!, key);

        return await BuildUploadMetadataAsync(key, url, size, null, cancellationToken);
    }

    /// <inheritdoc />
    public async Task<FileAddress?> UploadFileAsync(
        CloudFile file,
        CancellationToken cancellationToken = default
    )
    {
        var key = S3ObjectKeyHelper.BuildKey(file.PathToFile);
        var (stream, md5Value) = await CopyAndCalculateMd5Async(file.Stream, cancellationToken);
        await using (stream)
        {
            await UploadStreamAsync(stream, Bucket, key, cancellationToken);
        }

        return new FileAddress(S3ObjectKeyHelper.BuildAccessUrl(options.CdnHost!, key), md5Value);
    }

    /// <inheritdoc />
    public async Task<IFileMetadata> UploadAsync(
        byte[] bytes,
        List<string> path,
        string filename,
        CancellationToken cancellationToken = default
    )
    {
        ArgumentNullException.ThrowIfNull(bytes);

        var key = S3ObjectKeyHelper.BuildKey(path, filename);
        var hash = CalculateMd5(bytes);
        await using var stream = ApplicationTools.MemoryStreamManager.GetStream(bytes);
        await UploadStreamAsync(stream, Bucket, key, cancellationToken);
        var url = S3ObjectKeyHelper.BuildAccessUrl(options.CdnHost!, key);

        return await BuildUploadMetadataAsync(key, url, bytes.LongLength, hash, cancellationToken);
    }

    /// <inheritdoc />
    public async Task<Stream> DownloadAsync(
        string pathToFile,
        CancellationToken cancellationToken = default
    )
    {
        var key = S3ObjectKeyHelper.NormalizeKey(pathToFile);
        var response = await amazonS3.GetObjectAsync(Bucket, key, cancellationToken);
        return new S3ObjectResponseStream(response);
    }

    /// <inheritdoc />
    public async Task SaveAsAsync(
        string pathToFile,
        string path,
        CancellationToken cancellationToken = default
    )
    {
        if (string.IsNullOrEmpty(path))
        {
            throw new ArgumentNullException(nameof(path));
        }

        await using var stream = await DownloadAsync(pathToFile, cancellationToken);
        await using var fileStream = new FileStream(path, FileMode.Create, FileAccess.Write);
        await stream.CopyToAsync(fileStream, cancellationToken);
        await fileStream.FlushAsync(cancellationToken);
    }

    /// <inheritdoc />
    public async Task DeleteAsync(string? pathToFile, CancellationToken cancellationToken = default)
    {
        if (string.IsNullOrEmpty(pathToFile))
        {
            throw new ArgumentNullException(nameof(pathToFile));
        }

        var key = S3ObjectKeyHelper.NormalizeKey(pathToFile);
        try
        {
            await amazonS3.DeleteObjectAsync(Bucket, key, cancellationToken);
        }
        catch (Exception ex)
        {
            logger.LogError(ex, "COS 删除文件失败:{PathToFile}", key);
        }
    }

    /// <inheritdoc />
    public async Task<IList<FolderResult>> GetFolderListAsync(
        string path,
        CancellationToken cancellationToken = default
    )
    {
        var prefix = S3ObjectKeyHelper.NormalizeDirectoryPrefix(path);
        var results = new Dictionary<string, FolderResult>(StringComparer.OrdinalIgnoreCase);
        string? continuationToken = null;

        do
        {
            var response = await amazonS3.ListObjectsV2Async(
                new ListObjectsV2Request
                {
                    BucketName = Bucket,
                    Prefix = prefix,
                    Delimiter = "/",
                    ContinuationToken = continuationToken,
                },
                cancellationToken
            );

            foreach (var folder in response.CommonPrefixes ?? [])
            {
                var name = S3ObjectKeyHelper.GetDisplayName(folder, prefix);
                if (!string.IsNullOrWhiteSpace(name))
                {
                    results.TryAdd(
                        name,
                        new FolderResult
                        {
                            FileType = FileType.Directory,
                            LastUpdateTime = DateTime.MinValue,
                            Name = name,
                        }
                    );
                }
            }

            foreach (var item in response.S3Objects ?? [])
            {
                if (item.Key.Equals(prefix, StringComparison.OrdinalIgnoreCase))
                {
                    continue;
                }

                var name = S3ObjectKeyHelper.GetDisplayName(item.Key, prefix);
                if (!string.IsNullOrWhiteSpace(name))
                {
                    results[name] = new FolderResult
                    {
                        FileType = FileType.File,
                        LastUpdateTime = item.LastModified ?? DateTime.MinValue,
                        Name = name,
                        Size = item.Size,
                    };
                }
            }

            continuationToken =
                response.IsTruncated == true ? response.NextContinuationToken : null;
        } while (!string.IsNullOrEmpty(continuationToken));

        return results.Values.OrderBy(x => x.FileType).ThenBy(x => x.Name).ToList();
    }

    /// <inheritdoc />
    public async Task<FileInformation> GetFileInformationAsync(
        string pathToFile,
        CancellationToken cancellationToken = default
    )
    {
        var key = S3ObjectKeyHelper.NormalizeKey(pathToFile);
        var response = await amazonS3.GetObjectMetadataAsync(Bucket, key, cancellationToken);
        return new FileInformation
        {
            Length = response.ContentLength,
            Md5 = response.ETag?.Trim('"') ?? string.Empty,
            UploadTime = response.LastModified ?? DateTime.MinValue,
        };
    }

    /// <inheritdoc />
    public async Task<int> UploadDirectoryAsync(
        string localDirectory,
        string remoteDirectory,
        string? bucket = null,
        CancellationToken cancellationToken = default
    )
    {
        if (string.IsNullOrEmpty(localDirectory))
        {
            throw new ArgumentNullException(nameof(localDirectory));
        }

        if (string.IsNullOrEmpty(remoteDirectory))
        {
            throw new ArgumentNullException(nameof(remoteDirectory));
        }

        if (!Directory.Exists(localDirectory))
        {
            throw new DirectoryNotFoundException(localDirectory);
        }

        var effectiveBucket = bucket ?? Bucket;
        var successCount = 0;
        foreach (
            var filePath in Directory.EnumerateFiles(
                localDirectory,
                "*",
                SearchOption.AllDirectories
            )
        )
        {
            cancellationToken.ThrowIfCancellationRequested();
            var relativePath = Path.GetRelativePath(localDirectory, filePath);
            var key = S3ObjectKeyHelper.CombineRemotePath(remoteDirectory, relativePath);
            try
            {
                await ApplicationTools.RetryAsync(
                    async () =>
                    {
                        await using var stream = new FileStream(
                            filePath,
                            FileMode.Open,
                            FileAccess.Read
                        );
                        await UploadStreamAsync(stream, effectiveBucket, key, cancellationToken);
                    },
                    TimeSpan.FromSeconds(2),
                    specificExceptionTypes:
                    [
                        typeof(OperationCanceledException),
                        typeof(TaskCanceledException),
                    ]
                );
                successCount++;
            }
            catch (AggregateException ex)
            {
                logger.LogError(ex, "COS 文件上传失败,已放弃:{FilePath} -> {Key}", filePath, key);
            }
        }

        return successCount;
    }

    /// <inheritdoc />
    public async Task<int> CleanEmptyDirectoriesAsync(
        string? prefix = null,
        CancellationToken cancellationToken = default
    )
    {
        var stopwatch = Stopwatch.StartNew();
        var objects = new List<S3ListedObject>();
        string? continuationToken = null;
        var listPrefix = S3ObjectKeyHelper.NormalizeDirectoryPrefix(prefix ?? string.Empty);

        do
        {
            var request = new ListObjectsV2Request
            {
                BucketName = Bucket,
                Prefix = listPrefix,
                ContinuationToken = continuationToken,
            };
            var response = await ExecuteCosRequestAsync(
                () => amazonS3.ListObjectsV2Async(request, cancellationToken),
                cancellationToken
            );

            foreach (var item in response.S3Objects ?? [])
            {
                objects.Add(new S3ListedObject(item.Key, item.Size ?? 0));
            }

            continuationToken =
                response.IsTruncated == true ? response.NextContinuationToken : null;
        } while (!string.IsNullOrEmpty(continuationToken));

        var plan = S3DirectoryCleanupPlanner.BuildPlan(objects);
        var emptyDirectories = plan
            // 单次任务最多删除的空目录 marker 数量
            .EmptyDirectoryMarkers.Take(900)
            .ToList();
        if (emptyDirectories.Count == 0)
        {
            stopwatch.Stop();
            logger.LogInformation(
                "COS 未发现需要清理的空目录,Prefix:{Prefix},扫描对象:{ScannedObjectCount},候选 marker:{CandidateMarkerCount},耗时:{ElapsedMilliseconds}ms",
                listPrefix,
                plan.ScannedObjectCount,
                plan.CandidateMarkerCount,
                stopwatch.ElapsedMilliseconds
            );
            return 0;
        }

        var deletedCount = 0;
        var failedCount = 0;
        await Parallel.ForEachAsync(
            emptyDirectories,
            new ParallelOptions
            {
                // 删除空目录 marker 的并发数
                MaxDegreeOfParallelism = 4,
                CancellationToken = cancellationToken,
            },
            async (key, deleteCancellationToken) =>
            {
                try
                {
                    await ExecuteCosRequestAsync(
                        () =>
                            amazonS3.DeleteObjectAsync(
                                new DeleteObjectRequest { BucketName = Bucket, Key = key },
                                deleteCancellationToken
                            ),
                        deleteCancellationToken
                    );
                    Interlocked.Increment(ref deletedCount);
                }
                catch (Exception ex) when (ex is not OperationCanceledException)
                {
                    Interlocked.Increment(ref failedCount);
                    logger.LogError(ex, "COS 删除空目录失败,Key:{Key}", key);
                }
            }
        );

        stopwatch.Stop();
        logger.LogInformation(
            "COS 空目录清理完成,Prefix:{Prefix},扫描对象:{ScannedObjectCount},候选 marker:{CandidateMarkerCount},待删除:{RequestedDeleteCount},成功:{DeletedCount},失败:{FailedCount},耗时:{ElapsedMilliseconds}ms",
            listPrefix,
            plan.ScannedObjectCount,
            plan.CandidateMarkerCount,
            emptyDirectories.Count,
            deletedCount,
            failedCount,
            stopwatch.ElapsedMilliseconds
        );
        return deletedCount;
    }

    private async Task<T> ExecuteCosRequestAsync<T>(
        Func<Task<T>> request,
        CancellationToken cancellationToken = default
    )
    {
        try
        {
            return await ApplicationTools.RetryAsync(
                async () =>
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    try
                    {
                        return await request();
                    }
                    catch (Exception ex)
                        when (ex is not OperationCanceledException && !IsTransientCosException(ex))
                    {
                        throw new NonRetryableCosException(ex);
                    }
                },
                TimeSpan.FromSeconds(2),
                maxAttemptCount: 5,
                specificExceptionTypes:
                [
                    typeof(OperationCanceledException),
                    typeof(TaskCanceledException),
                    typeof(NonRetryableCosException),
                ]
            );
        }
        catch (NonRetryableCosException ex)
        {
            ExceptionDispatchInfo.Capture(ex.InnerException!).Throw();
            throw;
        }
    }

    private static bool IsTransientCosException(Exception exception)
    {
        if (exception is not AmazonS3Exception s3Exception)
        {
            return false;
        }

        if (
            s3Exception.StatusCode
            is HttpStatusCode.RequestTimeout
                or HttpStatusCode.TooManyRequests
                or HttpStatusCode.InternalServerError
                or HttpStatusCode.BadGateway
                or HttpStatusCode.ServiceUnavailable
                or HttpStatusCode.GatewayTimeout
        )
        {
            return true;
        }

        return s3Exception.ErrorCode
            is "SlowDown"
                or "Throttling"
                or "ThrottlingException"
                or "RequestTimeout"
                or "RequestLimitExceeded"
                or "TooManyRequests"
                or "TooManyRequestsException"
                or "InternalError"
                or "ServiceUnavailable";
    }

    private async Task UploadStreamAsync(
        Stream stream,
        string bucket,
        string key,
        CancellationToken cancellationToken = default
    )
    {
        if (stream.CanSeek)
        {
            stream.Position = 0;
        }

        var request = new PutObjectRequest
        {
            BucketName = bucket,
            Key = key,
            InputStream = stream,
            AutoCloseStream = false,
        };
        await amazonS3.PutObjectAsync(request, cancellationToken);
    }

    private async Task<IFileMetadata> BuildUploadMetadataAsync(
        string key,
        string url,
        long size,
        string? hash,
        CancellationToken cancellationToken = default
    )
    {
        var imageMetadata = await cloudInfiniteService.GetImageMetadataAsync(
            key,
            url,
            cancellationToken: cancellationToken
        );
        if (imageMetadata.HasValue)
        {
            return imageMetadata.Value with
            {
                Size = imageMetadata.Value.Size > 0 ? imageMetadata.Value.Size : size,
                Hash = hash,
            };
        }

        return new FileMetadata(url, size, hash);
    }

    private static async Task<(MemoryStream Stream, string Md5)> CopyAndCalculateMd5Async(
        Stream source,
        CancellationToken cancellationToken = default
    )
    {
        await using var memoryStream = ApplicationTools.MemoryStreamManager.GetStream();
        await source.CopyToAsync(memoryStream, cancellationToken);
        var bytes = memoryStream.ToArray();
        return (new MemoryStream(bytes), CalculateMd5(bytes));
    }

    private static string CalculateMd5(byte[] bytes)
    {
        var hash = MD5.HashData(bytes);
        return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
    }

    private sealed class NonRetryableCosException(Exception innerException)
        : Exception("COS 请求发生不可重试错误", innerException);

    private sealed class S3ObjectResponseStream(GetObjectResponse response) : Stream
    {
        private readonly Stream _stream = response.ResponseStream;

        public override bool CanRead => _stream.CanRead;

        public override bool CanSeek => _stream.CanSeek;

        public override bool CanWrite => _stream.CanWrite;

        public override long Length => _stream.Length;

        public override long Position
        {
            get => _stream.Position;
            set => _stream.Position = value;
        }

        public override void Flush()
        {
            _stream.Flush();
        }

        public override int Read(byte[] buffer, int offset, int count)
        {
            return _stream.Read(buffer, offset, count);
        }

        public override long Seek(long offset, SeekOrigin origin)
        {
            return _stream.Seek(offset, origin);
        }

        public override void SetLength(long value)
        {
            _stream.SetLength(value);
        }

        public override void Write(byte[] buffer, int offset, int count)
        {
            _stream.Write(buffer, offset, count);
        }

        public override ValueTask<int> ReadAsync(
            Memory<byte> buffer,
            CancellationToken cancellationToken = default
        )
        {
            return _stream.ReadAsync(buffer, cancellationToken);
        }

        protected override void Dispose(bool disposing)
        {
            if (disposing)
            {
                _stream.Dispose();
                response.Dispose();
            }

            base.Dispose(disposing);
        }

        public override async ValueTask DisposeAsync()
        {
            await _stream.DisposeAsync();
            response.Dispose();
            await base.DisposeAsync();
        }
    }
}
⚠⚠    以下内容为AI分析的结果,请根据实际情况进行判断。

代码解析

这是一个基于 Amazon S3 兼容 API(专门针对腾讯云 COS)的对象存储服务实现类。

整体架构

类定义

public class S3ObjectStorageService : IObjectStorageOperation
  • 实现了 IObjectStorageOperation 接口
  • 使用主构造函数注入依赖(C# 12 特性)

依赖注入

  1. IAmazonS3: AWS S3 SDK 客户端
  2. S3ObjectStorageOptions: 配置选项(Bucket、CDN域名等)
  3. ICloudInfiniteService: 云端图片处理服务
  4. ILogger: 日志记录器

核心功能模块

1. 文件上传

UploadAsync (Stream)

public async Task<IFileMetadata> UploadAsync(Stream stream, ...)
  • 从流上传文件
  • 构建对象键(Key)
  • 生成 CDN 访问 URL
  • 返回文件元数据(可能包含图片信息)

UploadFileAsync

public async Task<FileAddress?> UploadFileAsync(CloudFile file, ...)
  • 上传 CloudFile 对象
  • 自动计算 MD5 值
  • 返回访问地址和哈希值

UploadAsync (byte[])

  • 从字节数组上传
  • 本地计算 MD5

2. 文件下载

DownloadAsync

public async Task<Stream> DownloadAsync(string pathToFile, ...)
  • 返回 S3ObjectResponseStream(自定义流包装器)
  • 确保响应对象正确释放

SaveAsAsync

  • 下载文件并保存到本地路径

3. 文件管理

DeleteAsync

  • 删除指定对象
  • 异常只记录日志不抛出(防止删除失败影响业务)

GetFileInformationAsync

  • 获取文件元数据(大小、MD5、上传时间)

4. 目录操作

GetFolderListAsync

public async Task<IList<FolderResult>> GetFolderListAsync(...)
  • 使用 S3 ListObjectsV2 API
  • 通过 Delimiter="/" 模拟目录结构
  • 分页处理(ContinuationToken)
  • 返回排序后的文件/文件夹列表

UploadDirectoryAsync

public async Task<int> UploadDirectoryAsync(string localDirectory, ...)
  • 递归上传本地目录
  • 保持相对路径结构
  • 自动重试(排除取消异常)
  • 返回成功上传数量

CleanEmptyDirectoriesAsync

public async Task<int> CleanEmptyDirectoriesAsync(...)

复杂逻辑

  1. 列出所有对象
  2. 使用 S3DirectoryCleanupPlanner 分析空目录标记
  3. 限制单次删除 900 个
  4. 并行删除(并发度 4)
  5. 详细日志记录

关键技术点

1. 错误处理与重试机制

ExecuteCosRequestAsync

private async Task<T> ExecuteCosRequestAsync<T>(...)
  • 自动重试瞬态错误(超时、限流、5xx 错误)
  • 最多重试 5 次,间隔 2 秒
  • 非瞬态错误立即抛出

IsTransientCosException

判断是否为可重试错误:

  • HTTP 状态码:408, 429, 500, 502, 503, 504
  • 错误码:SlowDown, Throttling, RequestTimeout 等

NonRetryableCosException

  • 内部异常类,用于标记不可重试错误
  • 通过 ExceptionDispatchInfo.Capture().Throw() 保留原始堆栈

2. 流管理

S3ObjectResponseStream

private sealed class S3ObjectResponseStream : Stream
  • 装饰器模式包装 S3 响应流
  • 确保 GetObjectResponse 对象在流关闭时被释放
  • 实现完整的流接口和异步释放

3. 性能优化

MD5 计算

private static string CalculateMd5(byte[] bytes)
{
    var hash = MD5.HashData(bytes); // .NET 5+ 静态方法
    return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
}

流复制与 MD5

private static async Task<(MemoryStream, string)> CopyAndCalculateMd5Async(...)
  • 复制流到内存
  • 计算 MD5
  • 返回可重用的 MemoryStream

内存流管理

ApplicationTools.MemoryStreamManager.GetStream()
  • 使用对象池或 RecyclableMemoryStreamManager
  • 减少内存分配

4. 并发控制

await Parallel.ForEachAsync(
    emptyDirectories,
    new ParallelOptions { MaxDegreeOfParallelism = 4 },
    async (key, token) => { ... }
)
  • 限制并发度为 4
  • 使用 Interlocked.Increment 线程安全计数

辅助工具类(推测)

S3ObjectKeyHelper

  • BuildKey(): 构建对象键
  • NormalizeKey(): 标准化路径
  • BuildAccessUrl(): 生成 CDN URL
  • NormalizeDirectoryPrefix(): 处理目录前缀
  • GetDisplayName(): 提取显示名称
  • CombineRemotePath(): 合并远程路径

S3DirectoryCleanupPlanner

  • BuildPlan(): 分析对象列表,识别空目录标记

最佳实践体现

  1. 异步优先:全部使用 async/await
  2. 取消令牌支持:所有方法支持 CancellationToken
  3. 资源管理:使用 await using 确保流释放
  4. 防御性编程:参数校验、异常日志
  5. 可观测性:详细的结构化日志
  6. 弹性设计:自动重试 + 异常分类
  7. 性能优化:对象池、并行处理、流式读取

注意事项

安全性

  • CDN 域名从配置读取(options.CdnHost)
  • 生成的 URL 可能需要鉴权(未在代码中体现)

限制

  • 单次清理空目录最多 900 个(可能是腾讯云 COS 限制)
  • 目录上传不处理符号链接
  • 删除操作失败只记录日志(可能导致不一致)

腾讯云特性

  • 使用 ICloudInfiniteService 获取图片元数据(腾讯云数据万象)
  • 注释明确说明"基于腾讯云 COS S3 兼容 API"

这是一个生产级别的对象存储服务实现,代码质量高,适合企业级应用。

评论加载中...