feat: add storage quota limit for S3/R2 channels

- Add quota counter tracking (usedMB, fileCount) per channel
- Filter channels by quota threshold in fetchUploadConfig()
- Update quota on upload (endUpload) and delete
- Add /api/manage/quota API for stats and recalculation
- Fix null check in delete when file record not found
- Fix R2 channel name from hardcoded to actual channel.name
This commit is contained in:
axibayuit
2025-12-29 19:55:20 +08:00
parent 2c83f9062d
commit 3d77585ec5
6 changed files with 275 additions and 4 deletions

View File

@@ -2,6 +2,7 @@ import { S3Client, DeleteObjectCommand } from "@aws-sdk/client-s3";
import { purgeCFCache } from "../../../utils/purgeCache";
import { removeFileFromIndex, batchRemoveFilesFromIndex } from "../../../utils/indexManager.js";
import { getDatabase } from '../../../utils/databaseAdapter.js';
import { updateQuotaCounter } from '../../../upload/uploadTools.js';
// CORS 跨域响应头
const corsHeaders = {
@@ -125,6 +126,12 @@ async function deleteFile(env, fileId, cdnUrl, url) {
const db = getDatabase(env);
const img = await db.getWithMetadata(fileId);
// 如果文件记录不存在,直接返回成功(幂等删除)
if (!img) {
console.warn(`File ${fileId} not found in database, skipping delete`);
return true;
}
// 如果是R2渠道的图片需要删除R2中对应的图片
if (img.metadata?.Channel === 'CloudflareR2') {
const R2DataBase = env.img_r2;
@@ -136,6 +143,11 @@ async function deleteFile(env, fileId, cdnUrl, url) {
await deleteS3File(img);
}
// 更新容量计数器(在删除记录之前)
if (img.metadata?.ChannelName && img.metadata?.FileSize) {
await updateQuotaCounter(env, img.metadata.ChannelName, img.metadata.FileSize, 'subtract');
}
// 删除数据库中的记录
await db.delete(fileId);

View File

@@ -0,0 +1,175 @@
/**
* 容量配额管理 API
* GET: 获取各渠道容量统计
* POST: 重新统计容量(校准数据)
*/
import { getDatabase } from '../../utils/databaseAdapter.js';
import { getIndexInfo } from '../../utils/indexManager.js';
const corsHeaders = {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type, Authorization',
'Access-Control-Max-Age': '86400',
};
export async function onRequest(context) {
const { request, env } = context;
if (request.method === 'OPTIONS') {
return new Response(null, { headers: corsHeaders });
}
// GET: 获取容量统计
if (request.method === 'GET') {
return await getQuotaStats(env);
}
// POST: 重新统计容量
if (request.method === 'POST') {
return await recalculateQuota(context);
}
return new Response('Method not allowed', { status: 405, headers: corsHeaders });
}
// 获取各渠道容量统计
async function getQuotaStats(env) {
try {
const db = getDatabase(env);
const quotaStats = {};
// 获取所有 quota 记录
const listResult = await db.list({ prefix: 'manage@quota@' });
for (const item of listResult.keys) {
const channelName = item.name.replace('manage@quota@', '');
const quotaData = await db.get(item.name);
if (quotaData) {
quotaStats[channelName] = JSON.parse(quotaData);
}
}
return new Response(JSON.stringify({
success: true,
quotaStats
}), {
headers: { 'Content-Type': 'application/json', ...corsHeaders }
});
} catch (error) {
return new Response(JSON.stringify({
success: false,
error: error.message
}), {
status: 500,
headers: { 'Content-Type': 'application/json', ...corsHeaders }
});
}
}
// 重新统计各渠道容量
async function recalculateQuota(context) {
const { env } = context;
try {
const db = getDatabase(env);
// 获取索引信息,包含所有文件
const indexInfo = await getIndexInfo(context);
if (!indexInfo || !indexInfo.success) {
return new Response(JSON.stringify({
success: false,
error: 'Failed to get index info'
}), {
status: 500,
headers: { 'Content-Type': 'application/json', ...corsHeaders }
});
}
// 从索引中重新计算各渠道容量
const channelStats = {};
// 需要遍历索引中的所有文件
const allFiles = await getAllFilesFromIndex(context);
for (const file of allFiles) {
const channelName = file.metadata?.ChannelName;
const fileSize = parseFloat(file.metadata?.FileSize) || 0;
if (channelName) {
if (!channelStats[channelName]) {
channelStats[channelName] = { usedMB: 0, fileCount: 0 };
}
channelStats[channelName].usedMB += fileSize;
channelStats[channelName].fileCount += 1;
}
}
// 更新各渠道的 quota 记录
const now = Date.now();
for (const [channelName, stats] of Object.entries(channelStats)) {
const quotaKey = `manage@quota@${channelName}`;
const quotaData = {
usedMB: Math.round(stats.usedMB * 100) / 100, // 保留两位小数
fileCount: stats.fileCount,
lastUpdated: now,
recalculatedAt: now
};
await db.put(quotaKey, JSON.stringify(quotaData));
}
return new Response(JSON.stringify({
success: true,
message: 'Quota recalculated successfully',
channelStats
}), {
headers: { 'Content-Type': 'application/json', ...corsHeaders }
});
} catch (error) {
return new Response(JSON.stringify({
success: false,
error: error.message
}), {
status: 500,
headers: { 'Content-Type': 'application/json', ...corsHeaders }
});
}
}
// 从索引获取所有文件
async function getAllFilesFromIndex(context) {
const { env } = context;
const db = getDatabase(env);
const allFiles = [];
// 读取索引元数据
const metaData = await db.get('manage@index@meta');
if (!metaData) {
// 没有索引元数据,尝试读取旧格式索引
const oldIndex = await db.get('manage@index');
if (oldIndex) {
const index = JSON.parse(oldIndex);
return index.files || [];
}
return [];
}
const meta = JSON.parse(metaData);
const chunkCount = meta.chunkCount || 1;
// 读取所有分块
for (let i = 0; i < chunkCount; i++) {
const chunkKey = `manage@index_${i}`;
const chunkData = await db.get(chunkKey);
if (chunkData) {
const chunk = JSON.parse(chunkData);
allFiles.push(...chunk);
}
}
return allFiles;
}

View File

@@ -257,7 +257,10 @@ async function mergeR2ChunksInfo(context, uploadId, completedChunks, metadata) {
// 使用multipart info中的finalFileId更新metadata
const finalFileId = multipartInfo.key;
metadata.Channel = "CloudflareR2";
metadata.ChannelName = "R2_env";
// 从 R2 设置中获取渠道名称
const r2Settings = context.uploadConfig.cfr2;
const r2ChannelName = r2Settings.channels?.[0]?.name || "R2_env";
metadata.ChannelName = r2ChannelName;
metadata.FileSize = (totalSize / 1024 / 1024).toFixed(2);
// 清理multipart info

View File

@@ -232,7 +232,7 @@ async function uploadFileToCloudflareR2(context, fullId, metadata, returnLink) {
// 更新metadata
metadata.Channel = "CloudflareR2";
metadata.ChannelName = "R2_env";
metadata.ChannelName = r2Channel.name || "R2_env";
// 图像审查采用R2的publicUrl
const R2PublicUrl = r2Channel.publicUrl;
@@ -394,7 +394,7 @@ async function uploadFileToTelegram(context, fullId, metadata, fileExt, fileName
if (fileSize > CHUNK_SIZE) {
// 大文件分片上传
return await uploadLargeFileToTelegram(env, file, fullId, metadata, fileName, fileType, url, returnLink, tgBotToken, tgChatId, tgChannel);
return await uploadLargeFileToTelegram(context, file, fullId, metadata, fileName, fileType, returnLink, tgBotToken, tgChatId, tgChannel);
}
// 由于TG会把gif后缀的文件转为视频所以需要修改后缀名绕过限制

View File

@@ -177,7 +177,7 @@ export async function purgeCDNCache(env, cdnUrl, url, normalizedFolder) {
}
}
// 结束上传:清除缓存,维护索引
// 结束上传:清除缓存,维护索引,更新容量计数
export async function endUpload(context, fileId, metadata) {
const { env, url } = context;
@@ -188,6 +188,44 @@ export async function endUpload(context, fileId, metadata) {
// 更新文件索引
await addFileToIndex(context, fileId, metadata);
// 更新容量计数器
if (metadata.ChannelName && metadata.FileSize) {
await updateQuotaCounter(env, metadata.ChannelName, metadata.FileSize, 'add');
}
}
/**
* 更新渠道容量计数器
* @param {Object} env - 环境变量
* @param {string} channelName - 渠道名称
* @param {string|number} fileSizeMB - 文件大小(MB)
* @param {string} operation - 操作类型: 'add' 或 'subtract'
*/
export async function updateQuotaCounter(env, channelName, fileSizeMB, operation) {
if (!channelName) return;
try {
const db = getDatabase(env);
const quotaKey = `manage@quota@${channelName}`;
const quotaData = await db.get(quotaKey);
const quota = quotaData ? JSON.parse(quotaData) : { usedMB: 0, fileCount: 0 };
const sizeMB = parseFloat(fileSizeMB) || 0;
if (operation === 'add') {
quota.usedMB += sizeMB;
quota.fileCount += 1;
} else if (operation === 'subtract') {
quota.usedMB = Math.max(0, quota.usedMB - sizeMB);
quota.fileCount = Math.max(0, quota.fileCount - 1);
}
quota.lastUpdated = Date.now();
await db.put(quotaKey, JSON.stringify(quota));
} catch (error) {
console.error(`Failed to update quota counter for ${channelName}:`, error);
}
}
// 从 request 中解析 ip 地址

View File

@@ -4,6 +4,45 @@ import { getPageConfig } from '../api/manage/sysConfig/page';
import { getOthersConfig } from '../api/manage/sysConfig/others';
import { getDatabase } from './databaseAdapter.js';
/**
* 根据容量限制过滤渠道
* @param {Object} db - 数据库实例
* @param {Array} channels - 渠道列表
* @returns {Array} 过滤后的渠道列表
*/
async function filterChannelsByQuota(db, channels) {
const result = [];
for (const channel of channels) {
// 未启用容量限制,直接通过
if (!channel.quota?.enabled || !channel.quota?.limitGB) {
result.push(channel);
continue;
}
try {
const quotaKey = `manage@quota@${channel.name}`;
const quotaData = await db.get(quotaKey);
const quota = quotaData ? JSON.parse(quotaData) : { usedMB: 0, fileCount: 0 };
const usedGB = quota.usedMB / 1024;
const limitGB = channel.quota.limitGB;
const threshold = channel.quota.threshold || 95;
// 未超过阈值,渠道可用
if ((usedGB / limitGB) * 100 < threshold) {
result.push(channel);
} else {
console.log(`Channel ${channel.name} quota exceeded: ${usedGB.toFixed(2)}GB / ${limitGB}GB (${threshold}% threshold)`);
}
} catch (error) {
console.error(`Failed to check quota for channel ${channel.name}:`, error);
// 检查失败时保守处理,允许使用该渠道
result.push(channel);
}
}
return result;
}
export async function fetchUploadConfig(env) {
try {
const db = getDatabase(env);
@@ -13,6 +52,10 @@ export async function fetchUploadConfig(env) {
settings.cfr2.channels = settings.cfr2.channels.filter((channel) => channel.enabled);
settings.s3.channels = settings.s3.channels.filter((channel) => channel.enabled);
// 根据容量限制过滤渠道(仅 R2 和 S3
settings.cfr2.channels = await filterChannelsByQuota(db, settings.cfr2.channels);
settings.s3.channels = await filterChannelsByQuota(db, settings.s3.channels);
return settings;
} catch (error) {
console.error('Failed to fetch upload config:', error);