diff --git a/MIGRATION_GUIDE.md b/MIGRATION_GUIDE.md new file mode 100644 index 0000000..b3238be --- /dev/null +++ b/MIGRATION_GUIDE.md @@ -0,0 +1,191 @@ +# CloudFlare ImgBed KV 到 D1 数据库迁移指南 + +本指南将帮助您将现有的 KV 存储数据迁移到 D1 数据库。 + +## 迁移前准备 + +### 1. 配置 D1 数据库 + +首先,您需要在 Cloudflare 控制台创建一个 D1 数据库: + +```bash +# 创建 D1 数据库 +wrangler d1 create imgbed-database + +# 执行数据库初始化脚本 +wrangler d1 execute imgbed-database --file=./database/init.sql +``` + +### 2. 更新 wrangler.toml + +在您的 `wrangler.toml` 文件中添加 D1 数据库绑定: + +```toml +[[d1_databases]] +binding = "DB" +database_name = "imgbed-database" +database_id = "your-database-id" +``` + +### 3. 备份现有数据 + +在迁移前,强烈建议备份您的现有数据: + +1. 访问管理后台的系统设置 → 备份恢复 +2. 点击"备份数据"下载完整备份文件 +3. 保存备份文件到安全位置 + +## 迁移步骤 + +### 1. 检查迁移环境 + +访问迁移工具检查环境: +``` +GET /api/manage/migrate?action=check +``` + +确保返回结果中 `canMigrate` 为 `true`。 + +### 2. 查看迁移状态 + +查看当前数据统计: +``` +GET /api/manage/migrate?action=status +``` + +### 3. 执行迁移 + +开始数据迁移: +``` +GET /api/manage/migrate?action=migrate +``` + +迁移过程包括: +- 文件元数据迁移 +- 系统设置迁移 +- 索引操作迁移 + +## 迁移后验证 + +### 1. 检查数据完整性 + +- 访问管理后台,确认文件列表正常显示 +- 测试文件上传功能 +- 检查系统设置是否保持不变 + +### 2. 性能测试 + +- 测试文件访问速度 +- 验证管理功能正常工作 + +### 3. 功能验证 + +- 文件上传/删除 +- 备份恢复功能 +- 用户认证 +- API Token 管理 + +## 数据库结构说明 + +迁移后的 D1 数据库包含以下表: + +### files 表 +存储文件元数据,包含以下字段: +- `id`: 文件ID(主键) +- `value`: 文件值(用于分块文件) +- `metadata`: JSON格式的文件元数据 +- 其他索引字段:`file_name`, `file_type`, `timestamp` 等 + +### settings 表 +存储系统配置: +- `key`: 配置键(主键) +- `value`: 配置值 +- `category`: 配置分类 + +### index_operations 表 +存储索引操作记录: +- `id`: 操作ID(主键) +- `type`: 操作类型 +- `timestamp`: 时间戳 +- `data`: 操作数据 +- `processed`: 是否已处理 + +### index_metadata 表 +存储索引元数据: +- `key`: 元数据键 +- `last_updated`: 最后更新时间 +- `total_count`: 总文件数 +- `last_operation_id`: 最后操作ID + +### other_data 表 +存储其他数据(如黑名单IP等): +- `key`: 数据键 +- `value`: 数据值 +- `type`: 数据类型 + +## 兼容性说明 + +### 向后兼容 +- 系统会自动检测可用的数据库类型(D1 或 KV) +- 如果 D1 不可用,会自动回退到 KV 存储 +- 所有现有的 API 接口保持不变 + +### 环境变量 +- `DB`: D1 数据库绑定(新增) +- `img_url`: KV 存储绑定(保留作为备用) + +## 故障排除 + +### 常见问题 + +1. **迁移失败** + - 检查 D1 数据库是否正确配置 + - 确认数据库初始化脚本已执行 + - 查看迁移日志中的错误信息 + +2. **数据不完整** + - 检查迁移结果中的错误列表 + - 重新运行迁移(支持增量迁移) + - 使用备份文件恢复数据 + +3. **性能问题** + - D1 数据库查询比 KV 稍慢,这是正常的 + - 确保使用了适当的索引 + - 考虑优化查询语句 + +### 回滚方案 + +如果迁移后出现问题,可以: + +1. 临时禁用 D1 绑定,系统会自动回退到 KV +2. 使用备份文件恢复到迁移前状态 +3. 重新配置和测试 D1 数据库 + +## 性能优化建议 + +### 1. 索引优化 +D1 数据库已预设了必要的索引,包括: +- 文件时间戳索引 +- 目录索引 +- 文件类型索引 + +### 2. 查询优化 +- 使用分页查询大量数据 +- 避免全表扫描 +- 合理使用 WHERE 条件 + +### 3. 监控 +- 定期检查数据库性能 +- 监控查询执行时间 +- 关注错误日志 + +## 支持 + +如果在迁移过程中遇到问题,请: + +1. 查看浏览器控制台的错误信息 +2. 检查 Cloudflare Workers 的日志 +3. 确认所有配置正确 +4. 参考本指南的故障排除部分 + +迁移完成后,您的图床将使用更强大的 D1 数据库,享受更好的查询性能和数据管理能力。 diff --git a/database/init.sql b/database/init.sql new file mode 100644 index 0000000..b8b7e9f --- /dev/null +++ b/database/init.sql @@ -0,0 +1,127 @@ +-- CloudFlare ImgBed D1 Database Initialization Script +-- 这个脚本用于初始化D1数据库 + +-- 删除已存在的表(如果需要重新初始化) +-- 注意:在生产环境中使用时请谨慎 +-- DROP TABLE IF EXISTS files; +-- DROP TABLE IF EXISTS settings; +-- DROP TABLE IF EXISTS index_operations; +-- DROP TABLE IF EXISTS index_metadata; +-- DROP TABLE IF EXISTS other_data; + +-- 执行主要的数据库架构创建 +-- 这里会包含 schema.sql 的内容 + +-- 1. 文件表 - 存储文件元数据 +CREATE TABLE IF NOT EXISTS files ( + id TEXT PRIMARY KEY, + value TEXT, + metadata TEXT NOT NULL, + file_name TEXT, + file_type TEXT, + file_size TEXT, + upload_ip TEXT, + upload_address TEXT, + list_type TEXT, + timestamp INTEGER, + label TEXT, + directory TEXT, + channel TEXT, + channel_name TEXT, + tg_file_id TEXT, + tg_chat_id TEXT, + tg_bot_token TEXT, + is_chunked BOOLEAN DEFAULT FALSE, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- 2. 系统配置表 +CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + category TEXT, + description TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- 3. 索引操作表 +CREATE TABLE IF NOT EXISTS index_operations ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL, + timestamp INTEGER NOT NULL, + data TEXT NOT NULL, + processed BOOLEAN DEFAULT FALSE, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- 4. 索引元数据表 +CREATE TABLE IF NOT EXISTS index_metadata ( + key TEXT PRIMARY KEY, + last_updated INTEGER, + total_count INTEGER DEFAULT 0, + last_operation_id TEXT, + chunk_count INTEGER DEFAULT 0, + chunk_size INTEGER DEFAULT 0, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- 5. 其他数据表 +CREATE TABLE IF NOT EXISTS other_data ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + type TEXT, + description TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- 创建索引 +CREATE INDEX IF NOT EXISTS idx_files_timestamp ON files(timestamp DESC); +CREATE INDEX IF NOT EXISTS idx_files_directory ON files(directory); +CREATE INDEX IF NOT EXISTS idx_files_channel ON files(channel); +CREATE INDEX IF NOT EXISTS idx_files_file_type ON files(file_type); +CREATE INDEX IF NOT EXISTS idx_files_upload_ip ON files(upload_ip); +CREATE INDEX IF NOT EXISTS idx_files_created_at ON files(created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_settings_category ON settings(category); + +CREATE INDEX IF NOT EXISTS idx_index_operations_timestamp ON index_operations(timestamp); +CREATE INDEX IF NOT EXISTS idx_index_operations_processed ON index_operations(processed); +CREATE INDEX IF NOT EXISTS idx_index_operations_type ON index_operations(type); + +CREATE INDEX IF NOT EXISTS idx_other_data_type ON other_data(type); + +-- 创建触发器 +CREATE TRIGGER IF NOT EXISTS update_files_updated_at + AFTER UPDATE ON files + BEGIN + UPDATE files SET updated_at = CURRENT_TIMESTAMP WHERE id = NEW.id; + END; + +CREATE TRIGGER IF NOT EXISTS update_settings_updated_at + AFTER UPDATE ON settings + BEGIN + UPDATE settings SET updated_at = CURRENT_TIMESTAMP WHERE key = NEW.key; + END; + +CREATE TRIGGER IF NOT EXISTS update_index_metadata_updated_at + AFTER UPDATE ON index_metadata + BEGIN + UPDATE index_metadata SET updated_at = CURRENT_TIMESTAMP WHERE key = NEW.key; + END; + +CREATE TRIGGER IF NOT EXISTS update_other_data_updated_at + AFTER UPDATE ON other_data + BEGIN + UPDATE other_data SET updated_at = CURRENT_TIMESTAMP WHERE key = NEW.key; + END; + +-- 插入初始的索引元数据 +INSERT OR REPLACE INTO index_metadata (key, last_updated, total_count, last_operation_id) +VALUES ('main_index', 0, 0, NULL); + +-- 初始化完成 +-- 数据库已准备就绪,可以开始迁移数据 diff --git a/database/schema.sql b/database/schema.sql new file mode 100644 index 0000000..595845d --- /dev/null +++ b/database/schema.sql @@ -0,0 +1,116 @@ +-- CloudFlare ImgBed D1 Database Schema +-- 用于替代原有的KV存储 + +-- 1. 文件表 - 存储文件元数据 +CREATE TABLE IF NOT EXISTS files ( + id TEXT PRIMARY KEY, -- 文件ID (原KV的key) + value TEXT, -- 文件值 (对于分块文件,存储实际内容) + metadata TEXT NOT NULL, -- 文件元数据 (JSON格式) + file_name TEXT, -- 文件名 (从metadata中提取,便于查询) + file_type TEXT, -- 文件类型 (从metadata中提取) + file_size TEXT, -- 文件大小 (从metadata中提取) + upload_ip TEXT, -- 上传IP (从metadata中提取) + upload_address TEXT, -- 上传地址 (从metadata中提取) + list_type TEXT, -- 列表类型 (从metadata中提取) + timestamp INTEGER, -- 时间戳 (从metadata中提取,便于排序) + label TEXT, -- 标签 (从metadata中提取) + directory TEXT, -- 目录 (从metadata中提取,便于查询) + channel TEXT, -- 渠道 (从metadata中提取) + channel_name TEXT, -- 渠道名称 (从metadata中提取) + tg_file_id TEXT, -- Telegram文件ID (从metadata中提取) + tg_chat_id TEXT, -- Telegram聊天ID (从metadata中提取) + tg_bot_token TEXT, -- Telegram Bot Token (从metadata中提取) + is_chunked BOOLEAN DEFAULT FALSE, -- 是否为分块文件 + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- 为文件表创建索引 +CREATE INDEX IF NOT EXISTS idx_files_timestamp ON files(timestamp DESC); +CREATE INDEX IF NOT EXISTS idx_files_directory ON files(directory); +CREATE INDEX IF NOT EXISTS idx_files_channel ON files(channel); +CREATE INDEX IF NOT EXISTS idx_files_file_type ON files(file_type); +CREATE INDEX IF NOT EXISTS idx_files_upload_ip ON files(upload_ip); +CREATE INDEX IF NOT EXISTS idx_files_created_at ON files(created_at DESC); + +-- 2. 系统配置表 - 存储各种系统配置 +CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, -- 配置键 (原KV的key) + value TEXT NOT NULL, -- 配置值 (JSON格式) + category TEXT, -- 配置分类 (page, security, upload, others等) + description TEXT, -- 配置描述 + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- 为设置表创建索引 +CREATE INDEX IF NOT EXISTS idx_settings_category ON settings(category); + +-- 3. 索引操作表 - 存储原子操作记录 +CREATE TABLE IF NOT EXISTS index_operations ( + id TEXT PRIMARY KEY, -- 操作ID + type TEXT NOT NULL, -- 操作类型 (add, remove, move, batch_add等) + timestamp INTEGER NOT NULL, -- 时间戳 + data TEXT NOT NULL, -- 操作数据 (JSON格式) + processed BOOLEAN DEFAULT FALSE, -- 是否已处理 + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- 为索引操作表创建索引 +CREATE INDEX IF NOT EXISTS idx_index_operations_timestamp ON index_operations(timestamp); +CREATE INDEX IF NOT EXISTS idx_index_operations_processed ON index_operations(processed); +CREATE INDEX IF NOT EXISTS idx_index_operations_type ON index_operations(type); + +-- 4. 索引元数据表 - 存储索引的元信息 +CREATE TABLE IF NOT EXISTS index_metadata ( + key TEXT PRIMARY KEY, -- 元数据键 (如 'main_index') + last_updated INTEGER, -- 最后更新时间 + total_count INTEGER DEFAULT 0, -- 总文件数 + last_operation_id TEXT, -- 最后处理的操作ID + chunk_count INTEGER DEFAULT 0, -- 分块数量 (保留字段,D1中可能不需要) + chunk_size INTEGER DEFAULT 0, -- 分块大小 (保留字段) + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- 5. 其他数据表 - 存储黑名单IP等其他数据 +CREATE TABLE IF NOT EXISTS other_data ( + key TEXT PRIMARY KEY, -- 数据键 + value TEXT NOT NULL, -- 数据值 + type TEXT, -- 数据类型 (blacklist_ip, whitelist等) + description TEXT, -- 描述 + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- 为其他数据表创建索引 +CREATE INDEX IF NOT EXISTS idx_other_data_type ON other_data(type); + +-- 6. 创建触发器来自动更新 updated_at 字段 +-- 文件表触发器 +CREATE TRIGGER IF NOT EXISTS update_files_updated_at + AFTER UPDATE ON files + BEGIN + UPDATE files SET updated_at = CURRENT_TIMESTAMP WHERE id = NEW.id; + END; + +-- 设置表触发器 +CREATE TRIGGER IF NOT EXISTS update_settings_updated_at + AFTER UPDATE ON settings + BEGIN + UPDATE settings SET updated_at = CURRENT_TIMESTAMP WHERE key = NEW.key; + END; + +-- 索引元数据表触发器 +CREATE TRIGGER IF NOT EXISTS update_index_metadata_updated_at + AFTER UPDATE ON index_metadata + BEGIN + UPDATE index_metadata SET updated_at = CURRENT_TIMESTAMP WHERE key = NEW.key; + END; + +-- 其他数据表触发器 +CREATE TRIGGER IF NOT EXISTS update_other_data_updated_at + AFTER UPDATE ON other_data + BEGIN + UPDATE other_data SET updated_at = CURRENT_TIMESTAMP WHERE key = NEW.key; + END; diff --git a/functions/api/manage/apiTokens.js b/functions/api/manage/apiTokens.js index 48da31a..4ed0579 100644 --- a/functions/api/manage/apiTokens.js +++ b/functions/api/manage/apiTokens.js @@ -1,3 +1,5 @@ +import { getDatabase } from '../../utils/databaseAdapter.js'; + export async function onRequest(context) { // API Token管理,支持创建、删除、列出Token const { @@ -9,13 +11,13 @@ export async function onRequest(context) { data, } = context; - const kv = env.img_url + const db = getDatabase(env); const url = new URL(request.url) const method = request.method // GET - 获取所有Token列表 if (method === 'GET') { - const tokens = await getApiTokens(kv) + const tokens = await getApiTokens(db) return new Response(JSON.stringify(tokens), { headers: { 'content-type': 'application/json', @@ -37,7 +39,7 @@ export async function onRequest(context) { }) } - const token = await createApiToken(kv, name, permissions, owner) + const token = await createApiToken(db, name, permissions, owner) return new Response(JSON.stringify(token), { headers: { 'content-type': 'application/json', @@ -58,7 +60,7 @@ export async function onRequest(context) { }) } - const result = await deleteApiToken(kv, tokenId) + const result = await deleteApiToken(db, tokenId) return new Response(JSON.stringify(result), { headers: { 'content-type': 'application/json', @@ -80,7 +82,7 @@ export async function onRequest(context) { }) } - const result = await updateApiToken(kv, tokenId, permissions) + const result = await updateApiToken(db, tokenId, permissions) return new Response(JSON.stringify(result), { headers: { 'content-type': 'application/json', @@ -92,8 +94,8 @@ export async function onRequest(context) { } // 获取所有API Token -async function getApiTokens(kv) { - const settingsStr = await kv.get('manage@sysConfig@security') +async function getApiTokens(db) { + const settingsStr = await db.get('manage@sysConfig@security') const settings = settingsStr ? JSON.parse(settingsStr) : {} const tokens = settings.apiTokens?.tokens || {} @@ -115,8 +117,8 @@ async function getApiTokens(kv) { } // 创建新的API Token -async function createApiToken(kv, name, permissions, owner) { - const settingsStr = await kv.get('manage@sysConfig@security') +async function createApiToken(db, name, permissions, owner) { + const settingsStr = await db.get('manage@sysConfig@security') const settings = settingsStr ? JSON.parse(settingsStr) : {} if (!settings.apiTokens) { @@ -139,8 +141,8 @@ async function createApiToken(kv, name, permissions, owner) { settings.apiTokens.tokens[tokenId] = tokenData - // 保存到KV - await kv.put('manage@sysConfig@security', JSON.stringify(settings)) + // 保存到数据库 + await db.put('manage@sysConfig@security', JSON.stringify(settings)) return { id: tokenId, @@ -154,8 +156,8 @@ async function createApiToken(kv, name, permissions, owner) { } // 删除API Token -async function deleteApiToken(kv, tokenId) { - const settingsStr = await kv.get('manage@sysConfig@security') +async function deleteApiToken(db, tokenId) { + const settingsStr = await db.get('manage@sysConfig@security') const settings = settingsStr ? JSON.parse(settingsStr) : {} if (!settings.apiTokens?.tokens?.[tokenId]) { @@ -164,15 +166,15 @@ async function deleteApiToken(kv, tokenId) { delete settings.apiTokens.tokens[tokenId] - // 保存到KV - await kv.put('manage@sysConfig@security', JSON.stringify(settings)) + // 保存到数据库 + await db.put('manage@sysConfig@security', JSON.stringify(settings)) return { success: true, message: 'Token 已删除' } } // 更新API Token权限 -async function updateApiToken(kv, tokenId, permissions) { - const settingsStr = await kv.get('manage@sysConfig@security') +async function updateApiToken(db, tokenId, permissions) { + const settingsStr = await db.get('manage@sysConfig@security') const settings = settingsStr ? JSON.parse(settingsStr) : {} if (!settings.apiTokens?.tokens?.[tokenId]) { @@ -182,8 +184,8 @@ async function updateApiToken(kv, tokenId, permissions) { settings.apiTokens.tokens[tokenId].permissions = permissions settings.apiTokens.tokens[tokenId].updatedAt = new Date().toISOString() - // 保存到KV - await kv.put('manage@sysConfig@security', JSON.stringify(settings)) + // 保存到数据库 + await db.put('manage@sysConfig@security', JSON.stringify(settings)) return { success: true, @@ -208,8 +210,8 @@ function generateTokenId() { } // 根据Token获取权限(供其他API使用) -export async function getTokenPermissions(kv, token) { - const settingsStr = await kv.get('manage@sysConfig@security') +export async function getTokenPermissions(db, token) { + const settingsStr = await db.get('manage@sysConfig@security') const settings = settingsStr ? JSON.parse(settingsStr) : {} const tokens = settings.apiTokens?.tokens || {} diff --git a/functions/api/manage/cusConfig/blockipList.js b/functions/api/manage/cusConfig/blockipList.js index 55757e5..5cb5d22 100644 --- a/functions/api/manage/cusConfig/blockipList.js +++ b/functions/api/manage/cusConfig/blockipList.js @@ -1,3 +1,5 @@ +import { getDatabase } from '../../../utils/databaseAdapter.js'; + export async function onRequest(context) { // Contents of context object const { @@ -9,8 +11,8 @@ export async function onRequest(context) { data, // arbitrary space for passing data between middlewares } = context; try { - const kv = env.img_url; - const list = await kv.get("manage@blockipList"); + const db = getDatabase(env); + const list = await db.get("manage@blockipList"); if (list == null) { return new Response('', { status: 200 }); } else { diff --git a/functions/api/manage/delete/[[path]].js b/functions/api/manage/delete/[[path]].js index 8f94bc7..b89c652 100644 --- a/functions/api/manage/delete/[[path]].js +++ b/functions/api/manage/delete/[[path]].js @@ -1,6 +1,7 @@ import { S3Client, DeleteObjectCommand } from "@aws-sdk/client-s3"; import { purgeCFCache } from "../../../utils/purgeCache"; import { removeFileFromIndex, batchRemoveFilesFromIndex } from "../../../utils/indexManager.js"; +import { getDatabase } from '../../../utils/databaseAdapter'; export async function onRequest(context) { const { request, env, params, waitUntil } = context; @@ -104,7 +105,8 @@ export async function onRequest(context) { async function deleteFile(env, fileId, cdnUrl, url) { try { // 读取图片信息 - const img = await env.img_url.getWithMetadata(fileId); + const db = getDatabase(env); + const img = await db.getWithMetadata(fileId); // 如果是R2渠道的图片,需要删除R2中对应的图片 if (img.metadata?.Channel === 'CloudflareR2') { @@ -117,8 +119,8 @@ async function deleteFile(env, fileId, cdnUrl, url) { await deleteS3File(img); } - // 删除KV存储中的记录 - await env.img_url.delete(fileId); + // 删除数据库中的记录 + await db.delete(fileId); // 清除CDN缓存 await purgeCFCache(env, cdnUrl); diff --git a/functions/api/manage/migrate.js b/functions/api/manage/migrate.js new file mode 100644 index 0000000..7f810e0 --- /dev/null +++ b/functions/api/manage/migrate.js @@ -0,0 +1,262 @@ +/** + * 数据迁移工具 + * 用于将KV数据迁移到D1数据库 + */ + +import { getDatabase, checkDatabaseConfig } from '../../utils/databaseAdapter.js'; + +export async function onRequest(context) { + const { request, env } = context; + const url = new URL(request.url); + const action = url.searchParams.get('action'); + + try { + switch (action) { + case 'check': + return await handleCheck(env); + case 'migrate': + return await handleMigrate(env); + case 'status': + return await handleStatus(env); + default: + return new Response(JSON.stringify({ error: '不支持的操作' }), { + status: 400, + headers: { 'Content-Type': 'application/json' } + }); + } + } catch (error) { + console.error('迁移操作错误:', error); + return new Response(JSON.stringify({ error: '操作失败: ' + error.message }), { + status: 500, + headers: { 'Content-Type': 'application/json' } + }); + } +} + +// 检查迁移环境 +async function handleCheck(env) { + const dbConfig = checkDatabaseConfig(env); + + const result = { + hasKV: dbConfig.hasKV, + hasD1: dbConfig.hasD1, + canMigrate: dbConfig.hasKV && dbConfig.hasD1, + currentDatabase: dbConfig.usingD1 ? 'D1' : (dbConfig.usingKV ? 'KV' : 'None'), + message: '' + }; + + if (!result.canMigrate) { + if (!result.hasKV) { + result.message = '未找到KV存储,无法进行迁移'; + } else if (!result.hasD1) { + result.message = '未找到D1数据库,请先配置D1数据库'; + } + } else { + result.message = '环境检查通过,可以开始迁移'; + } + + return new Response(JSON.stringify(result), { + headers: { 'Content-Type': 'application/json' } + }); +} + +// 执行迁移 +async function handleMigrate(env) { + const dbConfig = checkDatabaseConfig(env); + + if (!dbConfig.hasKV || !dbConfig.hasD1) { + return new Response(JSON.stringify({ + error: '迁移环境不满足要求', + hasKV: dbConfig.hasKV, + hasD1: dbConfig.hasD1 + }), { + status: 400, + headers: { 'Content-Type': 'application/json' } + }); + } + + const migrationResult = { + startTime: new Date().toISOString(), + files: { migrated: 0, failed: 0, errors: [] }, + settings: { migrated: 0, failed: 0, errors: [] }, + operations: { migrated: 0, failed: 0, errors: [] }, + status: 'running' + }; + + try { + // 1. 迁移文件数据 + console.log('开始迁移文件数据...'); + await migrateFiles(env, migrationResult); + + // 2. 迁移系统设置 + console.log('开始迁移系统设置...'); + await migrateSettings(env, migrationResult); + + // 3. 迁移索引操作 + console.log('开始迁移索引操作...'); + await migrateIndexOperations(env, migrationResult); + + migrationResult.status = 'completed'; + migrationResult.endTime = new Date().toISOString(); + + } catch (error) { + migrationResult.status = 'failed'; + migrationResult.error = error.message; + migrationResult.endTime = new Date().toISOString(); + } + + return new Response(JSON.stringify(migrationResult), { + headers: { 'Content-Type': 'application/json' } + }); +} + +// 迁移文件数据 +async function migrateFiles(env, result) { + const db = getDatabase(env); + let cursor = null; + const batchSize = 100; + + while (true) { + const response = await env.img_url.list({ + limit: batchSize, + cursor: cursor + }); + + for (const item of response.keys) { + // 跳过管理相关的键 + if (item.name.startsWith('manage@') || item.name.startsWith('chunk_')) { + continue; + } + + try { + const fileData = await env.img_url.getWithMetadata(item.name); + + if (fileData && fileData.metadata) { + await db.putFile(item.name, fileData.value || '', { + metadata: fileData.metadata + }); + result.files.migrated++; + } + } catch (error) { + result.files.failed++; + result.files.errors.push({ + file: item.name, + error: error.message + }); + console.error(`迁移文件 ${item.name} 失败:`, error); + } + } + + cursor = response.cursor; + if (!cursor) break; + + // 添加延迟避免过载 + await new Promise(resolve => setTimeout(resolve, 10)); + } +} + +// 迁移系统设置 +async function migrateSettings(env, result) { + const db = getDatabase(env); + + const settingsList = await env.img_url.list({ prefix: 'manage@' }); + + for (const item of settingsList.keys) { + // 跳过索引相关的键 + if (item.name.startsWith('manage@index')) { + continue; + } + + try { + const value = await env.img_url.get(item.name); + if (value) { + await db.putSetting(item.name, value); + result.settings.migrated++; + } + } catch (error) { + result.settings.failed++; + result.settings.errors.push({ + setting: item.name, + error: error.message + }); + console.error(`迁移设置 ${item.name} 失败:`, error); + } + } +} + +// 迁移索引操作 +async function migrateIndexOperations(env, result) { + const db = getDatabase(env); + const operationPrefix = 'manage@index@operation_'; + + const operationsList = await env.img_url.list({ prefix: operationPrefix }); + + for (const item of operationsList.keys) { + try { + const operationData = await env.img_url.get(item.name); + if (operationData) { + const operation = JSON.parse(operationData); + const operationId = item.name.replace(operationPrefix, ''); + + await db.putIndexOperation(operationId, operation); + result.operations.migrated++; + } + } catch (error) { + result.operations.failed++; + result.operations.errors.push({ + operation: item.name, + error: error.message + }); + console.error(`迁移操作 ${item.name} 失败:`, error); + } + } +} + +// 获取迁移状态 +async function handleStatus(env) { + const dbConfig = checkDatabaseConfig(env); + + let fileCount = { kv: 0, d1: 0 }; + let settingCount = { kv: 0, d1: 0 }; + + try { + // 统计KV中的数据 + if (dbConfig.hasKV) { + const kvFiles = await env.img_url.list({ limit: 1000 }); + fileCount.kv = kvFiles.keys.filter(k => + !k.name.startsWith('manage@') && !k.name.startsWith('chunk_') + ).length; + + const kvSettings = await env.img_url.list({ prefix: 'manage@', limit: 1000 }); + settingCount.kv = kvSettings.keys.filter(k => + !k.name.startsWith('manage@index') + ).length; + } + + // 统计D1中的数据 + if (dbConfig.hasD1) { + const db = getDatabase(env); + + const fileCountStmt = db.db.prepare('SELECT COUNT(*) as count FROM files'); + const fileResult = await fileCountStmt.first(); + fileCount.d1 = fileResult.count; + + const settingCountStmt = db.db.prepare('SELECT COUNT(*) as count FROM settings'); + const settingResult = await settingCountStmt.first(); + settingCount.d1 = settingResult.count; + } + } catch (error) { + console.error('获取状态失败:', error); + } + + return new Response(JSON.stringify({ + database: dbConfig, + counts: { + files: fileCount, + settings: settingCount + }, + migrationNeeded: fileCount.kv > 0 && fileCount.d1 === 0 + }), { + headers: { 'Content-Type': 'application/json' } + }); +} diff --git a/functions/api/manage/sysConfig/backup.js b/functions/api/manage/sysConfig/backup.js index 7650b2b..83ac87d 100644 --- a/functions/api/manage/sysConfig/backup.js +++ b/functions/api/manage/sysConfig/backup.js @@ -1,4 +1,5 @@ import { readIndex } from '../../../utils/indexManager.js'; +import { getDatabase } from '../../../utils/databaseAdapter.js'; export async function onRequest(context) { const { request, env } = context; @@ -54,10 +55,11 @@ async function handleBackup(context) { const fileId = file.id; const metadata = file.metadata; - // 对于TelegramNew渠道且IsChunked为true的文件,需要从KV读取其值 + // 对于TelegramNew渠道且IsChunked为true的文件,需要从数据库读取其值 if (metadata.Channel === 'TelegramNew' && metadata.IsChunked === true) { try { - const fileData = await env.img_url.getWithMetadata(fileId); + const db = getDatabase(env); + const fileData = await db.getWithMetadata(fileId); backupData.data.files[fileId] = { metadata: metadata, value: fileData.value @@ -80,12 +82,13 @@ async function handleBackup(context) { } // 备份系统设置 - const settingsList = await env.img_url.list({ prefix: 'manage@' }); + const db = getDatabase(env); + const settingsList = await db.listSettings({ prefix: 'manage@' }); for (const key of settingsList.keys) { // 忽略索引文件 if (key.name.startsWith('manage@index')) continue; - const setting = await env.img_url.get(key.name); + const setting = key.value; if (setting) { backupData.data.settings[key.name] = setting; } @@ -131,16 +134,17 @@ async function handleRestore(request, env) { let restoredSettings = 0; // 恢复文件数据 + const db = getDatabase(env); for (const [key, fileData] of Object.entries(backupData.data.files)) { try { if (fileData.value) { // 对于有value的文件(如telegram分块文件),恢复完整数据 - await env.img_url.put(key, fileData.value, { + await db.put(key, fileData.value, { metadata: fileData.metadata }); } else if (fileData.metadata) { // 只恢复元数据 - await env.img_url.put(key, '', { + await db.put(key, '', { metadata: fileData.metadata }); } @@ -153,7 +157,7 @@ async function handleRestore(request, env) { // 恢复系统设置 for (const [key, value] of Object.entries(backupData.data.settings)) { try { - await env.img_url.put(key, value); + await db.put(key, value); restoredSettings++; } catch (error) { console.error(`恢复设置 ${key} 失败:`, error); diff --git a/functions/api/manage/sysConfig/others.js b/functions/api/manage/sysConfig/others.js index 8faee49..daceae9 100644 --- a/functions/api/manage/sysConfig/others.js +++ b/functions/api/manage/sysConfig/others.js @@ -1,3 +1,5 @@ +import { getDatabase } from '../../../utils/databaseAdapter'; + export async function onRequest(context) { // 其他设置相关,GET方法读取设置,POST方法保存设置 const { @@ -9,11 +11,11 @@ export async function onRequest(context) { data, // arbitrary space for passing data between middlewares } = context; - const kv = env.img_url + const db = getDatabase(env); // GET读取设置 if (request.method === 'GET') { - const settings = await getOthersConfig(kv, env) + const settings = await getOthersConfig(db, env) return new Response(JSON.stringify(settings), { headers: { @@ -27,8 +29,8 @@ export async function onRequest(context) { const body = await request.json() const settings = body - // 写入 KV - await kv.put('manage@sysConfig@others', JSON.stringify(settings)) + // 写入数据库 + await db.put('manage@sysConfig@others', JSON.stringify(settings)) return new Response(JSON.stringify(settings), { headers: { @@ -39,10 +41,10 @@ export async function onRequest(context) { } -export async function getOthersConfig(kv, env) { +export async function getOthersConfig(db, env) { const settings = {} - // 读取KV中的设置 - const settingsStr = await kv.get('manage@sysConfig@others') + // 读取数据库中的设置 + const settingsStr = await db.get('manage@sysConfig@others') const settingsKV = settingsStr ? JSON.parse(settingsStr) : {} // 远端遥测 diff --git a/functions/api/manage/sysConfig/page.js b/functions/api/manage/sysConfig/page.js index 11737e8..b3c6cc6 100644 --- a/functions/api/manage/sysConfig/page.js +++ b/functions/api/manage/sysConfig/page.js @@ -1,3 +1,5 @@ +import { getDatabase } from '../../../utils/databaseAdapter'; + export async function onRequest(context) { // 页面设置相关,GET方法读取设置,POST方法保存设置 const { @@ -9,11 +11,11 @@ export async function onRequest(context) { data, // arbitrary space for passing data between middlewares } = context; - const kv = env.img_url + const db = getDatabase(env); // GET读取设置 if (request.method === 'GET') { - const settings = await getPageConfig(kv, env) + const settings = await getPageConfig(db, env) return new Response(JSON.stringify(settings), { headers: { @@ -26,8 +28,8 @@ export async function onRequest(context) { if (request.method === 'POST') { const body = await request.json() const settings = body - // 写入 KV - await kv.put('manage@sysConfig@page', JSON.stringify(settings)) + // 写入数据库 + await db.put('manage@sysConfig@page', JSON.stringify(settings)) return new Response(JSON.stringify(settings), { headers: { @@ -38,10 +40,10 @@ export async function onRequest(context) { } -export async function getPageConfig(kv, env) { +export async function getPageConfig(db, env) { const settings = {} - // 读取KV中的设置 - const settingsStr = await kv.get('manage@sysConfig@page') + // 读取数据库中的设置 + const settingsStr = await db.get('manage@sysConfig@page') const settingsKV = settingsStr ? JSON.parse(settingsStr) : {} const config = [] diff --git a/functions/api/manage/sysConfig/security.js b/functions/api/manage/sysConfig/security.js index 72eddeb..41aef9a 100644 --- a/functions/api/manage/sysConfig/security.js +++ b/functions/api/manage/sysConfig/security.js @@ -1,3 +1,5 @@ +import { getDatabase } from '../../../utils/databaseAdapter'; + export async function onRequest(context) { // 安全设置相关,GET方法读取设置,POST方法保存设置 const { @@ -9,11 +11,11 @@ export async function onRequest(context) { data, // arbitrary space for passing data between middlewares } = context; - const kv = env.img_url + const db = getDatabase(env); // GET读取设置 if (request.method === 'GET') { - const settings = await getSecurityConfig(kv, env) + const settings = await getSecurityConfig(db, env) return new Response(JSON.stringify(settings), { headers: { @@ -27,8 +29,8 @@ export async function onRequest(context) { const body = await request.json() const settings = body - // 写入 KV - await kv.put('manage@sysConfig@security', JSON.stringify(settings)) + // 写入数据库 + await db.put('manage@sysConfig@security', JSON.stringify(settings)) return new Response(JSON.stringify(settings), { headers: { @@ -39,10 +41,10 @@ export async function onRequest(context) { } -export async function getSecurityConfig(kv, env) { +export async function getSecurityConfig(db, env) { const settings = {} - // 读取KV中的设置 - const settingsStr = await kv.get('manage@sysConfig@security') + // 读取数据库中的设置 + const settingsStr = await db.get('manage@sysConfig@security') const settingsKV = settingsStr ? JSON.parse(settingsStr) : {} // 认证管理 diff --git a/functions/api/manage/sysConfig/upload.js b/functions/api/manage/sysConfig/upload.js index dfefa40..a99328f 100644 --- a/functions/api/manage/sysConfig/upload.js +++ b/functions/api/manage/sysConfig/upload.js @@ -1,3 +1,5 @@ +import { getDatabase } from '../../../utils/databaseAdapter'; + export async function onRequest(context) { // 上传设置相关,GET方法读取设置,POST方法保存设置 const { @@ -9,11 +11,11 @@ export async function onRequest(context) { data, // arbitrary space for passing data between middlewares } = context; - const kv = env.img_url + const db = getDatabase(env); // GET读取设置 if (request.method === 'GET') { - const settings = await getUploadConfig(kv, env) + const settings = await getUploadConfig(db, env) return new Response(JSON.stringify(settings), { headers: { @@ -27,8 +29,8 @@ export async function onRequest(context) { const body = await request.json() const settings = body - // 写入 KV - await kv.put('manage@sysConfig@upload', JSON.stringify(settings)) + // 写入数据库 + await db.put('manage@sysConfig@upload', JSON.stringify(settings)) return new Response(JSON.stringify(settings), { headers: { @@ -39,10 +41,10 @@ export async function onRequest(context) { } -export async function getUploadConfig(kv, env) { +export async function getUploadConfig(db, env) { const settings = {} - // 读取KV中的设置 - const settingsStr = await kv.get('manage@sysConfig@upload') + // 读取数据库中的设置 + const settingsStr = await db.get('manage@sysConfig@upload') const settingsKV = settingsStr ? JSON.parse(settingsStr) : {} // =====================读取tg渠道配置===================== diff --git a/functions/file/[[path]].js b/functions/file/[[path]].js index 1a841ab..c78cc32 100644 --- a/functions/file/[[path]].js +++ b/functions/file/[[path]].js @@ -1,8 +1,9 @@ import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3"; import { fetchSecurityConfig } from "../utils/sysConfig"; import { TelegramAPI } from "../utils/telegramAPI"; -import { setCommonHeaders, setRangeHeaders, handleHeadRequest, getFileContent, isTgChannel, +import { setCommonHeaders, setRangeHeaders, handleHeadRequest, getFileContent, isTgChannel, returnWithCheck, return404, isDomainAllowed } from './fileTools'; +import { getDatabase } from '../utils/databaseAdapter'; export async function onRequest(context) { // Contents of context object @@ -39,8 +40,9 @@ export async function onRequest(context) { // Contents of context object return await returnBlockImg(url); } - // 从KV中获取图片记录 - const imgRecord = await env.img_url.getWithMetadata(fileId); + // 从数据库中获取图片记录 + const db = getDatabase(env); + const imgRecord = await db.getWithMetadata(fileId); if (!imgRecord) { return new Response('Error: Image Not Found', { status: 404 }); } diff --git a/functions/upload/index.js b/functions/upload/index.js index dcc50c6..408325a 100644 --- a/functions/upload/index.js +++ b/functions/upload/index.js @@ -1,11 +1,12 @@ import { userAuthCheck, UnauthorizedResponse } from "../utils/userAuth"; import { fetchUploadConfig, fetchSecurityConfig } from "../utils/sysConfig"; -import { createResponse, getUploadIp, getIPAddress, isExtValid, +import { createResponse, getUploadIp, getIPAddress, isExtValid, moderateContent, purgeCDNCache, isBlockedUploadIp, buildUniqueFileId, endUpload } from "./uploadTools"; import { initializeChunkedUpload, handleChunkUpload, uploadLargeFileToTelegram, handleCleanupRequest} from "./chunkUpload"; import { handleChunkMerge, checkMergeStatus } from "./chunkMerge"; import { TelegramAPI } from "../utils/telegramAPI"; import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3"; +import { getDatabase } from '../utils/databaseAdapter'; export async function onRequest(context) { // Contents of context object @@ -241,13 +242,14 @@ async function uploadFileToCloudflareR2(context, fullId, metadata, returnLink) { let moderateUrl = `${R2PublicUrl}/${fullId}`; metadata.Label = await moderateContent(env, moderateUrl); - // 写入KV数据库 + // 写入数据库 try { - await env.img_url.put(fullId, "", { + const db = getDatabase(env); + await db.put(fullId, "", { metadata: metadata, }); } catch (error) { - return createResponse('Error: Failed to write to KV database', { status: 500 }); + return createResponse('Error: Failed to write to database', { status: 500 }); } // 结束上传 @@ -347,11 +349,12 @@ async function uploadFileToS3(context, fullId, metadata, returnLink) { metadata.Label = await moderateContent(env, moderateUrl); } - // 写入 KV 数据库 + // 写入数据库 try { - await env.img_url.put(fullId, "", { metadata }); + const db = getDatabase(env); + await db.put(fullId, "", { metadata }); } catch { - return createResponse("Error: Failed to write to KV database", { status: 500 }); + return createResponse("Error: Failed to write to database", { status: 500 }); } // 结束上传 diff --git a/functions/utils/d1Database.js b/functions/utils/d1Database.js new file mode 100644 index 0000000..8cee73f --- /dev/null +++ b/functions/utils/d1Database.js @@ -0,0 +1,504 @@ +/** + * D1 数据库操作工具类 + * 用于替代原有的KV存储操作 + */ + +export class D1Database { + constructor(db) { + this.db = db; + } + + // ==================== 文件操作 ==================== + + /** + * 保存文件记录 (替代 KV.put) + * @param {string} fileId - 文件ID + * @param {string} value - 文件值 (对于分块文件) + * @param {Object} options - 选项,包含metadata + */ + async putFile(fileId, value = '', options = {}) { + const metadata = options.metadata || {}; + + // 从metadata中提取字段用于索引 + const extractedFields = this.extractMetadataFields(metadata); + + const stmt = this.db.prepare(` + INSERT OR REPLACE INTO files ( + id, value, metadata, file_name, file_type, file_size, + upload_ip, upload_address, list_type, timestamp, + label, directory, channel, channel_name, + tg_file_id, tg_chat_id, tg_bot_token, is_chunked + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + + return await stmt.bind( + fileId, + value, + JSON.stringify(metadata), + extractedFields.fileName, + extractedFields.fileType, + extractedFields.fileSize, + extractedFields.uploadIP, + extractedFields.uploadAddress, + extractedFields.listType, + extractedFields.timestamp, + extractedFields.label, + extractedFields.directory, + extractedFields.channel, + extractedFields.channelName, + extractedFields.tgFileId, + extractedFields.tgChatId, + extractedFields.tgBotToken, + extractedFields.isChunked + ).run(); + } + + /** + * 获取文件记录 (替代 KV.get) + * @param {string} fileId - 文件ID + * @returns {Object|null} 文件记录 + */ + async getFile(fileId) { + const stmt = this.db.prepare('SELECT * FROM files WHERE id = ?'); + const result = await stmt.bind(fileId).first(); + + if (!result) return null; + + return { + value: result.value, + metadata: JSON.parse(result.metadata || '{}') + }; + } + + /** + * 获取文件记录包含元数据 (替代 KV.getWithMetadata) + * @param {string} fileId - 文件ID + * @returns {Object|null} 文件记录 + */ + async getFileWithMetadata(fileId) { + return await this.getFile(fileId); + } + + /** + * 删除文件记录 (替代 KV.delete) + * @param {string} fileId - 文件ID + */ + async deleteFile(fileId) { + const stmt = this.db.prepare('DELETE FROM files WHERE id = ?'); + return await stmt.bind(fileId).run(); + } + + /** + * 列出文件 (替代 KV.list) + * @param {Object} options - 选项 + */ + async listFiles(options = {}) { + const { prefix = '', limit = 1000, cursor = null } = options; + + let query = 'SELECT id, metadata FROM files'; + let params = []; + + if (prefix) { + query += ' WHERE id LIKE ?'; + params.push(prefix + '%'); + } + + if (cursor) { + query += prefix ? ' AND' : ' WHERE'; + query += ' id > ?'; + params.push(cursor); + } + + query += ' ORDER BY id LIMIT ?'; + params.push(limit + 1); // 多取一个用于判断是否有下一页 + + const stmt = this.db.prepare(query); + const results = await stmt.bind(...params).all(); + + const hasMore = results.length > limit; + if (hasMore) { + results.pop(); // 移除多取的那一个 + } + + const keys = results.map(row => ({ + name: row.id, + metadata: JSON.parse(row.metadata || '{}') + })); + + return { + keys, + cursor: hasMore ? keys[keys.length - 1]?.name : null, + list_complete: !hasMore + }; + } + + // ==================== 设置操作 ==================== + + /** + * 保存设置 (替代 KV.put) + * @param {string} key - 设置键 + * @param {string} value - 设置值 + * @param {string} category - 设置分类 + */ + async putSetting(key, value, category = null) { + // 从key中推断category + if (!category && key.startsWith('manage@sysConfig@')) { + category = key.split('@')[2]; + } + + const stmt = this.db.prepare(` + INSERT OR REPLACE INTO settings (key, value, category) + VALUES (?, ?, ?) + `); + + return await stmt.bind(key, value, category).run(); + } + + /** + * 获取设置 (替代 KV.get) + * @param {string} key - 设置键 + * @returns {string|null} 设置值 + */ + async getSetting(key) { + const stmt = this.db.prepare('SELECT value FROM settings WHERE key = ?'); + const result = await stmt.bind(key).first(); + return result ? result.value : null; + } + + /** + * 删除设置 (替代 KV.delete) + * @param {string} key - 设置键 + */ + async deleteSetting(key) { + const stmt = this.db.prepare('DELETE FROM settings WHERE key = ?'); + return await stmt.bind(key).run(); + } + + /** + * 列出设置 (替代 KV.list) + * @param {Object} options - 选项 + */ + async listSettings(options = {}) { + const { prefix = '', limit = 1000 } = options; + + let query = 'SELECT key, value FROM settings'; + let params = []; + + if (prefix) { + query += ' WHERE key LIKE ?'; + params.push(prefix + '%'); + } + + query += ' ORDER BY key LIMIT ?'; + params.push(limit); + + const stmt = this.db.prepare(query); + const results = await stmt.bind(...params).all(); + + const keys = results.map(row => ({ + name: row.key, + value: row.value + })); + + return { keys }; + } + + // ==================== 索引操作 ==================== + + /** + * 保存索引操作记录 + * @param {string} operationId - 操作ID + * @param {Object} operation - 操作数据 + */ + async putIndexOperation(operationId, operation) { + const stmt = this.db.prepare(` + INSERT OR REPLACE INTO index_operations (id, type, timestamp, data) + VALUES (?, ?, ?, ?) + `); + + return await stmt.bind( + operationId, + operation.type, + operation.timestamp, + JSON.stringify(operation.data) + ).run(); + } + + /** + * 获取索引操作记录 + * @param {string} operationId - 操作ID + */ + async getIndexOperation(operationId) { + const stmt = this.db.prepare('SELECT * FROM index_operations WHERE id = ?'); + const result = await stmt.bind(operationId).first(); + + if (!result) return null; + + return { + type: result.type, + timestamp: result.timestamp, + data: JSON.parse(result.data) + }; + } + + /** + * 删除索引操作记录 + * @param {string} operationId - 操作ID + */ + async deleteIndexOperation(operationId) { + const stmt = this.db.prepare('DELETE FROM index_operations WHERE id = ?'); + return await stmt.bind(operationId).run(); + } + + /** + * 列出索引操作记录 + * @param {Object} options - 选项 + */ + async listIndexOperations(options = {}) { + const { limit = 1000, processed = null } = options; + + let query = 'SELECT * FROM index_operations'; + let params = []; + + if (processed !== null) { + query += ' WHERE processed = ?'; + params.push(processed); + } + + query += ' ORDER BY timestamp LIMIT ?'; + params.push(limit); + + const stmt = this.db.prepare(query); + const results = await stmt.bind(...params).all(); + + return results.map(row => ({ + id: row.id, + type: row.type, + timestamp: row.timestamp, + data: JSON.parse(row.data), + processed: row.processed + })); + } + + // ==================== 工具方法 ==================== + + /** + * 从metadata中提取字段用于索引 + * @param {Object} metadata - 元数据 + * @returns {Object} 提取的字段 + */ + extractMetadataFields(metadata) { + return { + fileName: metadata.FileName || null, + fileType: metadata.FileType || null, + fileSize: metadata.FileSize || null, + uploadIP: metadata.UploadIP || null, + uploadAddress: metadata.UploadAddress || null, + listType: metadata.ListType || null, + timestamp: metadata.TimeStamp || null, + label: metadata.Label || null, + directory: metadata.Directory || null, + channel: metadata.Channel || null, + channelName: metadata.ChannelName || null, + tgFileId: metadata.TgFileId || null, + tgChatId: metadata.TgChatId || null, + tgBotToken: metadata.TgBotToken || null, + isChunked: metadata.IsChunked || false + }; + } + + /** + * 通用的put方法,根据key类型自动选择存储位置 + * @param {string} key - 键 + * @param {string} value - 值 + * @param {Object} options - 选项 + */ + async put(key, value, options = {}) { + if (key.startsWith('manage@sysConfig@') || key.startsWith('manage@')) { + // 系统配置 + return await this.putSetting(key, value); + } else if (key.startsWith('manage@index@operation_')) { + // 索引操作 + const operationId = key.replace('manage@index@operation_', ''); + const operation = JSON.parse(value); + return await this.putIndexOperation(operationId, operation); + } else { + // 文件记录 + return await this.putFile(key, value, options); + } + } + + /** + * 通用的get方法,根据key类型自动选择获取位置 + * @param {string} key - 键 + */ + async get(key) { + if (key.startsWith('manage@sysConfig@') || key.startsWith('manage@')) { + // 系统配置 + return await this.getSetting(key); + } else if (key.startsWith('manage@index@operation_')) { + // 索引操作 + const operationId = key.replace('manage@index@operation_', ''); + const operation = await this.getIndexOperation(operationId); + return operation ? JSON.stringify(operation) : null; + } else { + // 文件记录 + const file = await this.getFile(key); + return file ? file.value : null; + } + } + + /** + * 通用的getWithMetadata方法 + * @param {string} key - 键 + */ + async getWithMetadata(key) { + if (key.startsWith('manage@sysConfig@') || key.startsWith('manage@')) { + // 系统配置没有metadata概念 + const value = await this.getSetting(key); + return value ? { value, metadata: {} } : null; + } else { + // 文件记录 + return await this.getFileWithMetadata(key); + } + } + + /** + * 通用的delete方法 + * @param {string} key - 键 + */ + async delete(key) { + if (key.startsWith('manage@sysConfig@') || key.startsWith('manage@')) { + // 系统配置 + return await this.deleteSetting(key); + } else if (key.startsWith('manage@index@operation_')) { + // 索引操作 + const operationId = key.replace('manage@index@operation_', ''); + return await this.deleteIndexOperation(operationId); + } else { + // 文件记录 + return await this.deleteFile(key); + } + } + + /** + * 通用的list方法 + * @param {Object} options - 选项 + */ + async list(options = {}) { + const { prefix = '' } = options; + + if (prefix.startsWith('manage@sysConfig@') || prefix.startsWith('manage@')) { + // 系统配置 + return await this.listSettings(options); + } else if (prefix.startsWith('manage@index@operation_')) { + // 索引操作 - 需要特殊处理 + const operations = await this.listIndexOperations(options); + const keys = operations.map(op => ({ + name: 'manage@index@operation_' + op.id + })); + return { keys }; + } else { + // 文件记录 + return await this.listFiles(options); + } + } +} + + /** + * 通用的put方法,根据key类型自动选择存储位置 + * @param {string} key - 键 + * @param {string} value - 值 + * @param {Object} options - 选项 + */ + async put(key, value, options = {}) { + if (key.startsWith('manage@sysConfig@') || key.startsWith('manage@')) { + // 系统配置 + return await this.putSetting(key, value); + } else if (key.startsWith('manage@index@operation_')) { + // 索引操作 + const operationId = key.replace('manage@index@operation_', ''); + const operation = JSON.parse(value); + return await this.putIndexOperation(operationId, operation); + } else { + // 文件记录 + return await this.putFile(key, value, options); + } + } + + /** + * 通用的get方法,根据key类型自动选择获取位置 + * @param {string} key - 键 + */ + async get(key) { + if (key.startsWith('manage@sysConfig@') || key.startsWith('manage@')) { + // 系统配置 + return await this.getSetting(key); + } else if (key.startsWith('manage@index@operation_')) { + // 索引操作 + const operationId = key.replace('manage@index@operation_', ''); + const operation = await this.getIndexOperation(operationId); + return operation ? JSON.stringify(operation) : null; + } else { + // 文件记录 + const file = await this.getFile(key); + return file ? file.value : null; + } + } + + /** + * 通用的getWithMetadata方法 + * @param {string} key - 键 + */ + async getWithMetadata(key) { + if (key.startsWith('manage@sysConfig@') || key.startsWith('manage@')) { + // 系统配置没有metadata概念 + const value = await this.getSetting(key); + return value ? { value, metadata: {} } : null; + } else { + // 文件记录 + return await this.getFileWithMetadata(key); + } + } + + /** + * 通用的delete方法 + * @param {string} key - 键 + */ + async delete(key) { + if (key.startsWith('manage@sysConfig@') || key.startsWith('manage@')) { + // 系统配置 + return await this.deleteSetting(key); + } else if (key.startsWith('manage@index@operation_')) { + // 索引操作 + const operationId = key.replace('manage@index@operation_', ''); + return await this.deleteIndexOperation(operationId); + } else { + // 文件记录 + return await this.deleteFile(key); + } + } + + /** + * 通用的list方法 + * @param {Object} options - 选项 + */ + async list(options = {}) { + const { prefix = '' } = options; + + if (prefix.startsWith('manage@sysConfig@') || prefix.startsWith('manage@')) { + // 系统配置 + return await this.listSettings(options); + } else if (prefix.startsWith('manage@index@operation_')) { + // 索引操作 - 需要特殊处理 + const operations = await this.listIndexOperations(options); + const keys = operations.map(op => ({ + name: 'manage@index@operation_' + op.id + })); + return { keys }; + } else { + // 文件记录 + return await this.listFiles(options); + } + } +} diff --git a/functions/utils/databaseAdapter.js b/functions/utils/databaseAdapter.js new file mode 100644 index 0000000..af84325 --- /dev/null +++ b/functions/utils/databaseAdapter.js @@ -0,0 +1,206 @@ +/** + * 数据库适配器 + * 提供统一的接口,可以在KV和D1之间切换 + */ + +import { D1Database } from './d1Database.js'; + +/** + * 创建数据库适配器 + * @param {Object} env - 环境变量 + * @returns {Object} 数据库适配器实例 + */ +export function createDatabaseAdapter(env) { + // 检查是否配置了D1数据库 + if (env.DB && typeof env.DB.prepare === 'function') { + // 使用D1数据库 + console.log('Using D1 Database'); + return new D1Database(env.DB); + } else if (env.img_url) { + // 回退到KV存储 + console.log('Using KV Storage (fallback)'); + return new KVAdapter(env.img_url); + } else { + throw new Error('No database configured. Please configure either D1 (env.DB) or KV (env.img_url)'); + } +} + +/** + * KV适配器类 + * 保持与原有KV接口的兼容性 + */ +class KVAdapter { + constructor(kv) { + this.kv = kv; + } + + // 直接代理到KV的方法 + async put(key, value, options = {}) { + return await this.kv.put(key, value, options); + } + + async get(key) { + return await this.kv.get(key); + } + + async getWithMetadata(key) { + return await this.kv.getWithMetadata(key); + } + + async delete(key) { + return await this.kv.delete(key); + } + + async list(options = {}) { + return await this.kv.list(options); + } + + // 为了兼容性,添加一些别名方法 + async putFile(fileId, value, options) { + return await this.put(fileId, value, options); + } + + async getFile(fileId) { + const result = await this.getWithMetadata(fileId); + return result; + } + + async getFileWithMetadata(fileId) { + return await this.getWithMetadata(fileId); + } + + async deleteFile(fileId) { + return await this.delete(fileId); + } + + async listFiles(options) { + return await this.list(options); + } + + async putSetting(key, value) { + return await this.put(key, value); + } + + async getSetting(key) { + return await this.get(key); + } + + async deleteSetting(key) { + return await this.delete(key); + } + + async listSettings(options) { + return await this.list(options); + } + + async putIndexOperation(operationId, operation) { + const key = 'manage@index@operation_' + operationId; + return await this.put(key, JSON.stringify(operation)); + } + + async getIndexOperation(operationId) { + const key = 'manage@index@operation_' + operationId; + const result = await this.get(key); + return result ? JSON.parse(result) : null; + } + + async deleteIndexOperation(operationId) { + const key = 'manage@index@operation_' + operationId; + return await this.delete(key); + } + + async listIndexOperations(options) { + const listOptions = { + ...options, + prefix: 'manage@index@operation_' + }; + const result = await this.list(listOptions); + + // 转换格式以匹配D1Database的返回格式 + const operations = []; + for (const item of result.keys) { + const operationData = await this.get(item.name); + if (operationData) { + const operation = JSON.parse(operationData); + operations.push({ + id: item.name.replace('manage@index@operation_', ''), + type: operation.type, + timestamp: operation.timestamp, + data: operation.data, + processed: false // KV中没有这个字段,默认为false + }); + } + } + + return operations; + } +} + +/** + * 获取数据库实例的便捷函数 + * 这个函数可以在整个应用中使用,确保一致的数据库访问 + * @param {Object} env - 环境变量 + * @returns {Object} 数据库实例 + */ +export function getDatabase(env) { + return createDatabaseAdapter(env); +} + +/** + * 检查数据库配置 + * @param {Object} env - 环境变量 + * @returns {Object} 配置信息 + */ +export function checkDatabaseConfig(env) { + const hasD1 = env.DB && typeof env.DB.prepare === 'function'; + const hasKV = env.img_url && typeof env.img_url.get === 'function'; + + return { + hasD1, + hasKV, + usingD1: hasD1, + usingKV: !hasD1 && hasKV, + configured: hasD1 || hasKV + }; +} + +/** + * 数据库健康检查 + * @param {Object} env - 环境变量 + * @returns {Promise} 健康检查结果 + */ +export async function healthCheck(env) { + const config = checkDatabaseConfig(env); + + if (!config.configured) { + return { + healthy: false, + error: 'No database configured', + config + }; + } + + try { + const db = getDatabase(env); + + if (config.usingD1) { + // D1健康检查 - 尝试查询一个简单的表 + const stmt = db.db.prepare('SELECT 1 as test'); + await stmt.first(); + } else { + // KV健康检查 - 尝试列出键 + await db.list({ limit: 1 }); + } + + return { + healthy: true, + config + }; + } catch (error) { + return { + healthy: false, + error: error.message, + config + }; + } +} diff --git a/functions/utils/indexManager.js b/functions/utils/indexManager.js index b34b983..983857a 100644 --- a/functions/utils/indexManager.js +++ b/functions/utils/indexManager.js @@ -1,38 +1,26 @@ -/* 索引管理器 */ +/* 索引管理器 - D1数据库版本 */ + +import { getDatabase } from './databaseAdapter.js'; /** - * 文件索引结构(分块存储): - * - * 索引元数据: - * - key: manage@index@meta - * - value: JSON.stringify(metadata) - * - metadata: { - * lastUpdated: 1640995200000, - * totalCount: 1000, - * lastOperationId: "operation_timestamp_uuid", - * chunkCount: 3, - * chunkSize: 10000 - * } - * - * 索引分块: - * - key: manage@index_${chunkId} (例如: manage@index_0, manage@index_1, ...) - * - value: JSON.stringify(filesChunk) - * - filesChunk: [ - * { - * id: "file_unique_id", - * metadata: {} - * }, - * ... - * ] - * - * 原子操作结构(保持不变): - * - key: manage@index@operation_${timestamp}_${uuid} - * - value: JSON.stringify(operation) + * 文件索引结构(D1数据库存储): + * + * 文件表: + * - 直接存储在 files 表中,包含所有文件信息和元数据 + * + * 索引元数据表: + * - 存储在 index_metadata 表中 + * - 包含 lastUpdated, totalCount, lastOperationId 等信息 + * + * 原子操作表: + * - 存储在 index_operations 表中 + * - 包含 id, type, timestamp, data, processed 等字段 * - operation: { * type: "add" | "remove" | "move" | "batch_add" | "batch_remove" | "batch_move", * timestamp: 1640995200000, * data: { - * // 根据操作类型包含不同的数据 + * fileId: "file_unique_id", + * metadata: {} * } * } */ @@ -55,8 +43,9 @@ export async function addFileToIndex(context, fileId, metadata = null) { try { if (metadata === null) { - // 如果未传入metadata,尝试从KV中获取 - const fileData = await env.img_url.getWithMetadata(fileId); + // 如果未传入metadata,尝试从数据库中获取 + const db = getDatabase(env); + const fileData = await db.getWithMetadata(fileId); metadata = fileData.metadata || {}; } @@ -390,8 +379,8 @@ export async function mergeOperationsToIndex(context, options = {}) { workingIndex.lastOperationId = processedOperationIds[processedOperationIds.length - 1]; } - // 保存更新后的索引(使用分块格式) - const saveSuccess = await saveChunkedIndex(context, workingIndex); + // 保存更新后的索引元数据 + const saveSuccess = await saveIndexMetadata(context, workingIndex); if (!saveSuccess) { console.error('Failed to save chunked index'); return { @@ -580,30 +569,24 @@ export async function rebuildIndex(context, progressCallback = null) { lastOperationId: null }; - // 分批读取所有文件 - while (true) { - const response = await env.img_url.list({ - limit: KV_LIST_LIMIT, - cursor: cursor - }); + // 从D1数据库读取所有文件 + const db = getDatabase(env); + const filesStmt = db.db.prepare('SELECT id, metadata FROM files WHERE timestamp IS NOT NULL ORDER BY timestamp DESC'); + const fileResults = await filesStmt.all(); - cursor = response.cursor; + for (const row of fileResults) { + try { + const metadata = JSON.parse(row.metadata || '{}'); - for (const item of response.keys) { - // 跳过管理相关的键 - if (item.name.startsWith('manage@') || item.name.startsWith('chunk_')) { - continue; - } - - // 跳过没有元数据的文件 - if (!item.metadata || !item.metadata.TimeStamp) { + // 跳过没有时间戳的文件 + if (!metadata.TimeStamp) { continue; } // 构建文件索引项 const fileItem = { - id: item.name, - metadata: item.metadata || {} + id: row.id, + metadata: metadata }; newIndex.files.push(fileItem); @@ -613,12 +596,9 @@ export async function rebuildIndex(context, progressCallback = null) { if (progressCallback && processedCount % 100 === 0) { progressCallback(processedCount); } + } catch (error) { + console.warn(`Failed to parse metadata for file ${row.id}:`, error); } - - if (!cursor) break; - - // 添加协作点 - await new Promise(resolve => setTimeout(resolve, 10)); } // 按时间戳倒序排序 @@ -626,8 +606,8 @@ export async function rebuildIndex(context, progressCallback = null) { newIndex.totalCount = newIndex.files.length; - // 保存新索引(使用分块格式) - const saveSuccess = await saveChunkedIndex(context, newIndex); + // 保存新索引元数据 + const saveSuccess = await saveIndexMetadata(context, newIndex); if (!saveSuccess) { console.error('Failed to save chunked index during rebuild'); return { @@ -742,9 +722,9 @@ async function recordOperation(context, type, data) { timestamp: Date.now(), data }; - - const operationKey = OPERATION_KEY_PREFIX + operationId; - await env.img_url.put(operationKey, JSON.stringify(operation)); + + const db = getDatabase(env); + await db.putIndexOperation(operationId, operation); return operationId; } @@ -761,33 +741,18 @@ async function getAllPendingOperations(context, lastOperationId = null) { let cursor = null; try { - while (true) { - const response = await env.img_url.list({ - prefix: OPERATION_KEY_PREFIX, - limit: KV_LIST_LIMIT, - cursor: cursor - }); - - for (const item of response.keys) { - // 如果指定了lastOperationId,跳过已处理的操作 - if (lastOperationId && item.name <= OPERATION_KEY_PREFIX + lastOperationId) { - continue; - } - - try { - const operationData = await env.img_url.get(item.name); - if (operationData) { - const operation = JSON.parse(operationData); - operation.id = item.name.substring(OPERATION_KEY_PREFIX.length); - operations.push(operation); - } - } catch (error) { - console.warn(`Failed to parse operation ${item.name}:`, error); - } + const db = getDatabase(env); + const allOperations = await db.listIndexOperations({ + processed: false, + limit: 10000 // 获取所有未处理的操作 + }); + + // 如果指定了lastOperationId,过滤已处理的操作 + for (const operation of allOperations) { + if (lastOperationId && operation.id <= lastOperationId) { + continue; } - - cursor = response.cursor; - if (!cursor) break; + operations.push(operation); } } catch (error) { console.error('Error getting pending operations:', error); @@ -1053,8 +1018,8 @@ export async function deleteAllOperations(context) { async function getIndex(context) { const { waitUntil } = context; try { - // 首先尝试加载分块索引 - const index = await loadChunkedIndex(context); + // 首先尝试加载索引 + const index = await loadIndexFromDatabase(context); if (index.success) { return index; } else { @@ -1167,93 +1132,66 @@ async function promiseLimit(tasks, concurrency = BATCH_SIZE) { * @param {Object} index - 完整的索引对象 * @returns {Promise} 是否保存成功 */ -async function saveChunkedIndex(context, index) { +async function saveIndexMetadata(context, index) { const { env } = context; - + try { - const files = index.files || []; - const chunks = []; - - // 将文件数组分块 - for (let i = 0; i < files.length; i += INDEX_CHUNK_SIZE) { - const chunk = files.slice(i, i + INDEX_CHUNK_SIZE); - chunks.push(chunk); - } - - // 保存索引元数据 - const metadata = { - lastUpdated: index.lastUpdated, - totalCount: index.totalCount, - lastOperationId: index.lastOperationId, - chunkCount: chunks.length, - chunkSize: INDEX_CHUNK_SIZE - }; - - await env.img_url.put(INDEX_META_KEY, JSON.stringify(metadata)); - - // 保存各个分块 - const savePromises = chunks.map((chunk, chunkId) => { - const chunkKey = `${INDEX_KEY}_${chunkId}`; - return env.img_url.put(chunkKey, JSON.stringify(chunk)); - }); - - await Promise.all(savePromises); - - console.log(`Saved chunked index: ${chunks.length} chunks, ${files.length} total files`); + const db = getDatabase(env); + + // 保存索引元数据到index_metadata表 + const stmt = db.db.prepare(` + INSERT OR REPLACE INTO index_metadata (key, last_updated, total_count, last_operation_id) + VALUES (?, ?, ?, ?) + `); + + await stmt.bind( + 'main_index', + index.lastUpdated, + index.totalCount, + index.lastOperationId + ).run(); + + console.log(`Saved index metadata: ${index.totalCount} total files, last updated: ${index.lastUpdated}`); return true; - + } catch (error) { - console.error('Error saving chunked index:', error); + console.error('Error saving index metadata:', error); return false; } } /** - * 从KV存储加载分块索引 + * 从D1数据库加载索引 * @param {Object} context - 上下文对象,包含 env * @returns {Promise} 完整的索引对象 */ -async function loadChunkedIndex(context) { +async function loadIndexFromDatabase(context) { const { env } = context; - + try { + const db = getDatabase(env); + // 首先获取元数据 - const metadataStr = await env.img_url.get(INDEX_META_KEY); - if (!metadataStr) { + const metadataStmt = db.db.prepare('SELECT * FROM index_metadata WHERE key = ?'); + const metadata = await metadataStmt.bind('main_index').first(); + + if (!metadata) { throw new Error('Index metadata not found'); } - - const metadata = JSON.parse(metadataStr); - const files = []; - - // 并行加载所有分块 - const loadPromises = []; - for (let chunkId = 0; chunkId < metadata.chunkCount; chunkId++) { - const chunkKey = `${INDEX_KEY}_${chunkId}`; - loadPromises.push( - env.img_url.get(chunkKey).then(chunkStr => { - if (chunkStr) { - return JSON.parse(chunkStr); - } - return []; - }) - ); - } - - const chunks = await Promise.all(loadPromises); - - // 合并所有分块 - chunks.forEach(chunk => { - if (Array.isArray(chunk)) { - files.push(...chunk); - } - }); - + // 从files表直接查询所有文件 + const filesStmt = db.db.prepare('SELECT id, metadata FROM files ORDER BY timestamp DESC'); + const fileResults = await filesStmt.all(); + + const files = fileResults.map(row => ({ + id: row.id, + metadata: JSON.parse(row.metadata || '{}') + })); + const index = { files, - lastUpdated: metadata.lastUpdated, - totalCount: metadata.totalCount, - lastOperationId: metadata.lastOperationId, + lastUpdated: metadata.last_updated, + totalCount: metadata.total_count, + lastOperationId: metadata.last_operation_id, success: true }; diff --git a/functions/utils/middleware.js b/functions/utils/middleware.js index 6a74409..a14216f 100644 --- a/functions/utils/middleware.js +++ b/functions/utils/middleware.js @@ -112,18 +112,21 @@ async function fetchSampleRate(context) { } } -// 检查 KV 是否配置,文件索引是否存在 -export async function checkKVConfig(context) { +import { checkDatabaseConfig } from './databaseAdapter.js'; + +// 检查数据库是否配置,文件索引是否存在 +export async function checkDatabaseConfig(context) { const { env, waitUntil } = context; - // 检查 img_url KV 绑定是否存在 - if (typeof env.img_url == "undefined" || env.img_url == null) { + const dbConfig = checkDatabaseConfig(env); + + if (!dbConfig.configured) { return new Response( JSON.stringify({ success: false, - error: "KV 数据库未配置 / KV not configured", - message: "img_url KV 绑定未找到,请检查您的 KV 配置。 / img_url KV binding not found, please check your KV configuration." - }), + error: "数据库未配置 / Database not configured", + message: "请配置 D1 数据库 (env.DB) 或 KV 存储 (env.img_url)。 / Please configure D1 database (env.DB) or KV storage (env.img_url)." + }), { status: 500, headers: { @@ -135,4 +138,7 @@ export async function checkKVConfig(context) { // 继续执行 return context.next(); -} \ No newline at end of file +} + +// 保持向后兼容性的别名 +export const checkKVConfig = checkDatabaseConfig; \ No newline at end of file diff --git a/functions/utils/sysConfig.js b/functions/utils/sysConfig.js index b5feb37..5fe24f2 100644 --- a/functions/utils/sysConfig.js +++ b/functions/utils/sysConfig.js @@ -2,10 +2,11 @@ import { getUploadConfig } from '../api/manage/sysConfig/upload'; import { getSecurityConfig } from '../api/manage/sysConfig/security'; import { getPageConfig } from '../api/manage/sysConfig/page'; import { getOthersConfig } from '../api/manage/sysConfig/others'; +import { getDatabase } from './databaseAdapter'; export async function fetchUploadConfig(env) { - const kv = env.img_url; - const settings = await getUploadConfig(kv, env); + const db = getDatabase(env); + const settings = await getUploadConfig(db, env); // 去除 已禁用 的渠道 settings.telegram.channels = settings.telegram.channels.filter((channel) => channel.enabled); settings.cfr2.channels = settings.cfr2.channels.filter((channel) => channel.enabled); @@ -15,19 +16,19 @@ export async function fetchUploadConfig(env) { } export async function fetchSecurityConfig(env) { - const kv = env.img_url; - const settings = await getSecurityConfig(kv, env); + const db = getDatabase(env); + const settings = await getSecurityConfig(db, env); return settings; } export async function fetchPageConfig(env) { - const kv = env.img_url; - const settings = await getPageConfig(kv, env); + const db = getDatabase(env); + const settings = await getPageConfig(db, env); return settings; } export async function fetchOthersConfig(env) { - const kv = env.img_url; - const settings = await getOthersConfig(kv, env); + const db = getDatabase(env); + const settings = await getOthersConfig(db, env); return settings; } \ No newline at end of file diff --git a/wrangler.toml b/wrangler.toml new file mode 100644 index 0000000..d5767c2 --- /dev/null +++ b/wrangler.toml @@ -0,0 +1,4 @@ +[[d1_databases]] +binding = "DB" +database_name = "imgbed-database" +database_id = "your-database-id" \ No newline at end of file