using System.Diagnostics;
using System.Net;
using System.Runtime.ExceptionServices;
using System.Security.Cryptography;
using Amazon.S3;
using Amazon.S3.Model;
using Dpz.Core.Entity.Base.PublicStruct;
using Dpz.Core.Infrastructure;
using Dpz.Core.Shard.Service;
using Microsoft.Extensions.Logging;
namespace Dpz.Core.Service.ObjectStorage.Services.Impl;
/// <summary>
/// 基于腾讯云 COS S3 兼容 API 的对象存储操作
/// </summary>
public class S3ObjectStorageService(
IAmazonS3 amazonS3,
S3ObjectStorageOptions options,
ICloudInfiniteService cloudInfiniteService,
ILogger<S3ObjectStorageService> logger
) : IObjectStorageOperation
{
/// <inheritdoc />
public string Bucket => options.Bucket ?? throw new BusinessException("S3 bucket is null");
/// <inheritdoc />
public async Task<IFileMetadata> UploadAsync(
Stream stream,
ICollection<string> path,
string filename,
CancellationToken cancellationToken = default
)
{
ArgumentNullException.ThrowIfNull(stream);
var key = S3ObjectKeyHelper.BuildKey(path, filename);
var size = stream.CanSeek ? stream.Length : 0;
await UploadStreamAsync(stream, Bucket, key, cancellationToken);
var url = S3ObjectKeyHelper.BuildAccessUrl(options.CdnHost!, key);
return await BuildUploadMetadataAsync(key, url, size, null, cancellationToken);
}
/// <inheritdoc />
public async Task<FileAddress?> UploadFileAsync(
CloudFile file,
CancellationToken cancellationToken = default
)
{
var key = S3ObjectKeyHelper.BuildKey(file.PathToFile);
var (stream, md5Value) = await CopyAndCalculateMd5Async(file.Stream, cancellationToken);
await using (stream)
{
await UploadStreamAsync(stream, Bucket, key, cancellationToken);
}
return new FileAddress(S3ObjectKeyHelper.BuildAccessUrl(options.CdnHost!, key), md5Value);
}
/// <inheritdoc />
public async Task<IFileMetadata> UploadAsync(
byte[] bytes,
List<string> path,
string filename,
CancellationToken cancellationToken = default
)
{
ArgumentNullException.ThrowIfNull(bytes);
var key = S3ObjectKeyHelper.BuildKey(path, filename);
var hash = CalculateMd5(bytes);
await using var stream = ApplicationTools.MemoryStreamManager.GetStream(bytes);
await UploadStreamAsync(stream, Bucket, key, cancellationToken);
var url = S3ObjectKeyHelper.BuildAccessUrl(options.CdnHost!, key);
return await BuildUploadMetadataAsync(key, url, bytes.LongLength, hash, cancellationToken);
}
/// <inheritdoc />
public async Task<Stream> DownloadAsync(
string pathToFile,
CancellationToken cancellationToken = default
)
{
var key = S3ObjectKeyHelper.NormalizeKey(pathToFile);
var response = await amazonS3.GetObjectAsync(Bucket, key, cancellationToken);
return new S3ObjectResponseStream(response);
}
/// <inheritdoc />
public async Task SaveAsAsync(
string pathToFile,
string path,
CancellationToken cancellationToken = default
)
{
if (string.IsNullOrEmpty(path))
{
throw new ArgumentNullException(nameof(path));
}
await using var stream = await DownloadAsync(pathToFile, cancellationToken);
await using var fileStream = new FileStream(path, FileMode.Create, FileAccess.Write);
await stream.CopyToAsync(fileStream, cancellationToken);
await fileStream.FlushAsync(cancellationToken);
}
/// <inheritdoc />
public async Task DeleteAsync(string? pathToFile, CancellationToken cancellationToken = default)
{
if (string.IsNullOrEmpty(pathToFile))
{
throw new ArgumentNullException(nameof(pathToFile));
}
var key = S3ObjectKeyHelper.NormalizeKey(pathToFile);
try
{
await amazonS3.DeleteObjectAsync(Bucket, key, cancellationToken);
}
catch (Exception ex)
{
logger.LogError(ex, "COS 删除文件失败:{PathToFile}", key);
}
}
/// <inheritdoc />
public async Task<IList<FolderResult>> GetFolderListAsync(
string path,
CancellationToken cancellationToken = default
)
{
var prefix = S3ObjectKeyHelper.NormalizeDirectoryPrefix(path);
var results = new Dictionary<string, FolderResult>(StringComparer.OrdinalIgnoreCase);
string? continuationToken = null;
do
{
var response = await amazonS3.ListObjectsV2Async(
new ListObjectsV2Request
{
BucketName = Bucket,
Prefix = prefix,
Delimiter = "/",
ContinuationToken = continuationToken,
},
cancellationToken
);
foreach (var folder in response.CommonPrefixes ?? [])
{
var name = S3ObjectKeyHelper.GetDisplayName(folder, prefix);
if (!string.IsNullOrWhiteSpace(name))
{
results.TryAdd(
name,
new FolderResult
{
FileType = FileType.Directory,
LastUpdateTime = DateTime.MinValue,
Name = name,
}
);
}
}
foreach (var item in response.S3Objects ?? [])
{
if (item.Key.Equals(prefix, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var name = S3ObjectKeyHelper.GetDisplayName(item.Key, prefix);
if (!string.IsNullOrWhiteSpace(name))
{
results[name] = new FolderResult
{
FileType = FileType.File,
LastUpdateTime = item.LastModified ?? DateTime.MinValue,
Name = name,
Size = item.Size,
};
}
}
continuationToken =
response.IsTruncated == true ? response.NextContinuationToken : null;
} while (!string.IsNullOrEmpty(continuationToken));
return results.Values.OrderBy(x => x.FileType).ThenBy(x => x.Name).ToList();
}
/// <inheritdoc />
public async Task<FileInformation> GetFileInformationAsync(
string pathToFile,
CancellationToken cancellationToken = default
)
{
var key = S3ObjectKeyHelper.NormalizeKey(pathToFile);
var response = await amazonS3.GetObjectMetadataAsync(Bucket, key, cancellationToken);
return new FileInformation
{
Length = response.ContentLength,
Md5 = response.ETag?.Trim('"') ?? string.Empty,
UploadTime = response.LastModified ?? DateTime.MinValue,
};
}
/// <inheritdoc />
public async Task<int> UploadDirectoryAsync(
string localDirectory,
string remoteDirectory,
string? bucket = null,
CancellationToken cancellationToken = default
)
{
if (string.IsNullOrEmpty(localDirectory))
{
throw new ArgumentNullException(nameof(localDirectory));
}
if (string.IsNullOrEmpty(remoteDirectory))
{
throw new ArgumentNullException(nameof(remoteDirectory));
}
if (!Directory.Exists(localDirectory))
{
throw new DirectoryNotFoundException(localDirectory);
}
var effectiveBucket = bucket ?? Bucket;
var successCount = 0;
foreach (
var filePath in Directory.EnumerateFiles(
localDirectory,
"*",
SearchOption.AllDirectories
)
)
{
cancellationToken.ThrowIfCancellationRequested();
var relativePath = Path.GetRelativePath(localDirectory, filePath);
var key = S3ObjectKeyHelper.CombineRemotePath(remoteDirectory, relativePath);
try
{
await ApplicationTools.RetryAsync(
async () =>
{
await using var stream = new FileStream(
filePath,
FileMode.Open,
FileAccess.Read
);
await UploadStreamAsync(stream, effectiveBucket, key, cancellationToken);
},
TimeSpan.FromSeconds(2),
specificExceptionTypes:
[
typeof(OperationCanceledException),
typeof(TaskCanceledException),
]
);
successCount++;
}
catch (AggregateException ex)
{
logger.LogError(ex, "COS 文件上传失败,已放弃:{FilePath} -> {Key}", filePath, key);
}
}
return successCount;
}
/// <inheritdoc />
public async Task<int> CleanEmptyDirectoriesAsync(
string? prefix = null,
CancellationToken cancellationToken = default
)
{
var stopwatch = Stopwatch.StartNew();
var objects = new List<S3ListedObject>();
string? continuationToken = null;
var listPrefix = S3ObjectKeyHelper.NormalizeDirectoryPrefix(prefix ?? string.Empty);
do
{
var request = new ListObjectsV2Request
{
BucketName = Bucket,
Prefix = listPrefix,
ContinuationToken = continuationToken,
};
var response = await ExecuteCosRequestAsync(
() => amazonS3.ListObjectsV2Async(request, cancellationToken),
cancellationToken
);
foreach (var item in response.S3Objects ?? [])
{
objects.Add(new S3ListedObject(item.Key, item.Size ?? 0));
}
continuationToken =
response.IsTruncated == true ? response.NextContinuationToken : null;
} while (!string.IsNullOrEmpty(continuationToken));
var plan = S3DirectoryCleanupPlanner.BuildPlan(objects);
var emptyDirectories = plan
// 单次任务最多删除的空目录 marker 数量
.EmptyDirectoryMarkers.Take(900)
.ToList();
if (emptyDirectories.Count == 0)
{
stopwatch.Stop();
logger.LogInformation(
"COS 未发现需要清理的空目录,Prefix:{Prefix},扫描对象:{ScannedObjectCount},候选 marker:{CandidateMarkerCount},耗时:{ElapsedMilliseconds}ms",
listPrefix,
plan.ScannedObjectCount,
plan.CandidateMarkerCount,
stopwatch.ElapsedMilliseconds
);
return 0;
}
var deletedCount = 0;
var failedCount = 0;
await Parallel.ForEachAsync(
emptyDirectories,
new ParallelOptions
{
// 删除空目录 marker 的并发数
MaxDegreeOfParallelism = 4,
CancellationToken = cancellationToken,
},
async (key, deleteCancellationToken) =>
{
try
{
await ExecuteCosRequestAsync(
() =>
amazonS3.DeleteObjectAsync(
new DeleteObjectRequest { BucketName = Bucket, Key = key },
deleteCancellationToken
),
deleteCancellationToken
);
Interlocked.Increment(ref deletedCount);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
Interlocked.Increment(ref failedCount);
logger.LogError(ex, "COS 删除空目录失败,Key:{Key}", key);
}
}
);
stopwatch.Stop();
logger.LogInformation(
"COS 空目录清理完成,Prefix:{Prefix},扫描对象:{ScannedObjectCount},候选 marker:{CandidateMarkerCount},待删除:{RequestedDeleteCount},成功:{DeletedCount},失败:{FailedCount},耗时:{ElapsedMilliseconds}ms",
listPrefix,
plan.ScannedObjectCount,
plan.CandidateMarkerCount,
emptyDirectories.Count,
deletedCount,
failedCount,
stopwatch.ElapsedMilliseconds
);
return deletedCount;
}
private async Task<T> ExecuteCosRequestAsync<T>(
Func<Task<T>> request,
CancellationToken cancellationToken = default
)
{
try
{
return await ApplicationTools.RetryAsync(
async () =>
{
cancellationToken.ThrowIfCancellationRequested();
try
{
return await request();
}
catch (Exception ex)
when (ex is not OperationCanceledException && !IsTransientCosException(ex))
{
throw new NonRetryableCosException(ex);
}
},
TimeSpan.FromSeconds(2),
maxAttemptCount: 5,
specificExceptionTypes:
[
typeof(OperationCanceledException),
typeof(TaskCanceledException),
typeof(NonRetryableCosException),
]
);
}
catch (NonRetryableCosException ex)
{
ExceptionDispatchInfo.Capture(ex.InnerException!).Throw();
throw;
}
}
private static bool IsTransientCosException(Exception exception)
{
if (exception is not AmazonS3Exception s3Exception)
{
return false;
}
if (
s3Exception.StatusCode
is HttpStatusCode.RequestTimeout
or HttpStatusCode.TooManyRequests
or HttpStatusCode.InternalServerError
or HttpStatusCode.BadGateway
or HttpStatusCode.ServiceUnavailable
or HttpStatusCode.GatewayTimeout
)
{
return true;
}
return s3Exception.ErrorCode
is "SlowDown"
or "Throttling"
or "ThrottlingException"
or "RequestTimeout"
or "RequestLimitExceeded"
or "TooManyRequests"
or "TooManyRequestsException"
or "InternalError"
or "ServiceUnavailable";
}
private async Task UploadStreamAsync(
Stream stream,
string bucket,
string key,
CancellationToken cancellationToken = default
)
{
if (stream.CanSeek)
{
stream.Position = 0;
}
var request = new PutObjectRequest
{
BucketName = bucket,
Key = key,
InputStream = stream,
AutoCloseStream = false,
};
await amazonS3.PutObjectAsync(request, cancellationToken);
}
private async Task<IFileMetadata> BuildUploadMetadataAsync(
string key,
string url,
long size,
string? hash,
CancellationToken cancellationToken = default
)
{
var imageMetadata = await cloudInfiniteService.GetImageMetadataAsync(
key,
url,
cancellationToken: cancellationToken
);
if (imageMetadata.HasValue)
{
return imageMetadata.Value with
{
Size = imageMetadata.Value.Size > 0 ? imageMetadata.Value.Size : size,
Hash = hash,
};
}
return new FileMetadata(url, size, hash);
}
private static async Task<(MemoryStream Stream, string Md5)> CopyAndCalculateMd5Async(
Stream source,
CancellationToken cancellationToken = default
)
{
await using var memoryStream = ApplicationTools.MemoryStreamManager.GetStream();
await source.CopyToAsync(memoryStream, cancellationToken);
var bytes = memoryStream.ToArray();
return (new MemoryStream(bytes), CalculateMd5(bytes));
}
private static string CalculateMd5(byte[] bytes)
{
var hash = MD5.HashData(bytes);
return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
}
private sealed class NonRetryableCosException(Exception innerException)
: Exception("COS 请求发生不可重试错误", innerException);
private sealed class S3ObjectResponseStream(GetObjectResponse response) : Stream
{
private readonly Stream _stream = response.ResponseStream;
public override bool CanRead => _stream.CanRead;
public override bool CanSeek => _stream.CanSeek;
public override bool CanWrite => _stream.CanWrite;
public override long Length => _stream.Length;
public override long Position
{
get => _stream.Position;
set => _stream.Position = value;
}
public override void Flush()
{
_stream.Flush();
}
public override int Read(byte[] buffer, int offset, int count)
{
return _stream.Read(buffer, offset, count);
}
public override long Seek(long offset, SeekOrigin origin)
{
return _stream.Seek(offset, origin);
}
public override void SetLength(long value)
{
_stream.SetLength(value);
}
public override void Write(byte[] buffer, int offset, int count)
{
_stream.Write(buffer, offset, count);
}
public override ValueTask<int> ReadAsync(
Memory<byte> buffer,
CancellationToken cancellationToken = default
)
{
return _stream.ReadAsync(buffer, cancellationToken);
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
_stream.Dispose();
response.Dispose();
}
base.Dispose(disposing);
}
public override async ValueTask DisposeAsync()
{
await _stream.DisposeAsync();
response.Dispose();
await base.DisposeAsync();
}
}
}
⚠⚠ 以下内容为AI分析的结果,请根据实际情况进行判断。
代码解析
这是一个基于 Amazon S3 兼容 API(专门针对腾讯云 COS)的对象存储服务实现类。
整体架构
类定义
public class S3ObjectStorageService : IObjectStorageOperation
- 实现了
IObjectStorageOperation接口 - 使用主构造函数注入依赖(C# 12 特性)
依赖注入
- IAmazonS3: AWS S3 SDK 客户端
- S3ObjectStorageOptions: 配置选项(Bucket、CDN域名等)
- ICloudInfiniteService: 云端图片处理服务
- ILogger: 日志记录器
核心功能模块
1. 文件上传
UploadAsync (Stream)
public async Task<IFileMetadata> UploadAsync(Stream stream, ...)
- 从流上传文件
- 构建对象键(Key)
- 生成 CDN 访问 URL
- 返回文件元数据(可能包含图片信息)
UploadFileAsync
public async Task<FileAddress?> UploadFileAsync(CloudFile file, ...)
- 上传
CloudFile对象 - 自动计算 MD5 值
- 返回访问地址和哈希值
UploadAsync (byte[])
- 从字节数组上传
- 本地计算 MD5
2. 文件下载
DownloadAsync
public async Task<Stream> DownloadAsync(string pathToFile, ...)
- 返回 S3ObjectResponseStream(自定义流包装器)
- 确保响应对象正确释放
SaveAsAsync
- 下载文件并保存到本地路径
3. 文件管理
DeleteAsync
- 删除指定对象
- 异常只记录日志不抛出(防止删除失败影响业务)
GetFileInformationAsync
- 获取文件元数据(大小、MD5、上传时间)
4. 目录操作
GetFolderListAsync
public async Task<IList<FolderResult>> GetFolderListAsync(...)
- 使用 S3
ListObjectsV2API - 通过 Delimiter="/" 模拟目录结构
- 分页处理(ContinuationToken)
- 返回排序后的文件/文件夹列表
UploadDirectoryAsync
public async Task<int> UploadDirectoryAsync(string localDirectory, ...)
- 递归上传本地目录
- 保持相对路径结构
- 自动重试(排除取消异常)
- 返回成功上传数量
CleanEmptyDirectoriesAsync ⭐
public async Task<int> CleanEmptyDirectoriesAsync(...)
复杂逻辑:
- 列出所有对象
- 使用
S3DirectoryCleanupPlanner分析空目录标记 - 限制单次删除 900 个
- 并行删除(并发度 4)
- 详细日志记录
关键技术点
1. 错误处理与重试机制
ExecuteCosRequestAsync
private async Task<T> ExecuteCosRequestAsync<T>(...)
- 自动重试瞬态错误(超时、限流、5xx 错误)
- 最多重试 5 次,间隔 2 秒
- 非瞬态错误立即抛出
IsTransientCosException
判断是否为可重试错误:
- HTTP 状态码:408, 429, 500, 502, 503, 504
- 错误码:SlowDown, Throttling, RequestTimeout 等
NonRetryableCosException
- 内部异常类,用于标记不可重试错误
- 通过
ExceptionDispatchInfo.Capture().Throw()保留原始堆栈
2. 流管理
S3ObjectResponseStream
private sealed class S3ObjectResponseStream : Stream
- 装饰器模式包装 S3 响应流
- 确保
GetObjectResponse对象在流关闭时被释放 - 实现完整的流接口和异步释放
3. 性能优化
MD5 计算
private static string CalculateMd5(byte[] bytes)
{
var hash = MD5.HashData(bytes); // .NET 5+ 静态方法
return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
}
流复制与 MD5
private static async Task<(MemoryStream, string)> CopyAndCalculateMd5Async(...)
- 复制流到内存
- 计算 MD5
- 返回可重用的 MemoryStream
内存流管理
ApplicationTools.MemoryStreamManager.GetStream()
- 使用对象池或 RecyclableMemoryStreamManager
- 减少内存分配
4. 并发控制
await Parallel.ForEachAsync(
emptyDirectories,
new ParallelOptions { MaxDegreeOfParallelism = 4 },
async (key, token) => { ... }
)
- 限制并发度为 4
- 使用
Interlocked.Increment线程安全计数
辅助工具类(推测)
S3ObjectKeyHelper
BuildKey(): 构建对象键NormalizeKey(): 标准化路径BuildAccessUrl(): 生成 CDN URLNormalizeDirectoryPrefix(): 处理目录前缀GetDisplayName(): 提取显示名称CombineRemotePath(): 合并远程路径
S3DirectoryCleanupPlanner
BuildPlan(): 分析对象列表,识别空目录标记
最佳实践体现
- ✅ 异步优先:全部使用 async/await
- ✅ 取消令牌支持:所有方法支持 CancellationToken
- ✅ 资源管理:使用
await using确保流释放 - ✅ 防御性编程:参数校验、异常日志
- ✅ 可观测性:详细的结构化日志
- ✅ 弹性设计:自动重试 + 异常分类
- ✅ 性能优化:对象池、并行处理、流式读取
注意事项
安全性
- CDN 域名从配置读取(options.CdnHost)
- 生成的 URL 可能需要鉴权(未在代码中体现)
限制
- 单次清理空目录最多 900 个(可能是腾讯云 COS 限制)
- 目录上传不处理符号链接
- 删除操作失败只记录日志(可能导致不一致)
腾讯云特性
- 使用
ICloudInfiniteService获取图片元数据(腾讯云数据万象) - 注释明确说明"基于腾讯云 COS S3 兼容 API"
这是一个生产级别的对象存储服务实现,代码质量高,适合企业级应用。
AI 正在分析代码…
评论加载中...