admin/yunrui_asset
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 修复BUG

Browse Source
This commit is contained in:
zhang1106
2026-03-16 17:07:20 +08:00
parent 8dcbe9948b
commit e5d20820dc
26 changed files with 2363 additions and 1004 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/utils/autoBackupScheduler.js
+157 -74
View File
@@ -1,7 +1,3 @@
/**
* 自动备份调度器模块
* 使用 node-cron 实现定时自动备份功能
*/
const cron = require('node-cron');
const path = require('path');
@@ -9,28 +5,23 @@ const fs = require('fs');
const { createBackup, createIncrementalBackup, getBackupPath } = require('./backup');
const { uploadToRemote } = require('./remoteBackup');
const { getEnabledTargets, getGlobalSettings } = require('./remoteBackupConfig');
const { createLogEntry, updateLogStatus } = require('./backupLog');
// 全局调度器存储
const schedulers = new Map();
// 备份设置文件路径
const SETTINGS_FILE = path.join(__dirname, '..', 'config', 'auto-backup-settings.json');
// 默认设置
const DEFAULT_SETTINGS = {
enabled: false,
cronExpression: '0 2 * * *', // 每天凌晨 2 点
cronExpression: '0 2 * * *',
description: '自动备份',
backupType: 'full', // 'full' 或 'incremental'
backupType: 'full',
includeFiles: true,
compress: true,
maxCount: 30,
maxAgeDays: 90,
};
/**
* 加载备份设置
*/
function loadSettings() {
try {
if (fs.existsSync(SETTINGS_FILE)) {
@@ -43,9 +34,6 @@ function loadSettings() {
return { ...DEFAULT_SETTINGS };
}
/**
* 保存备份设置
*/
function saveSettings(settings) {
try {
const configDir = path.dirname(SETTINGS_FILE);
@@ -60,23 +48,46 @@ function saveSettings(settings) {
}
}
/**
* 验证 Cron 表达式
*/
function validateCronExpression(expression) {
return cron.validate(expression);
}
/**
* 将中文时间转换为 Cron 表达式
*/
function timeToCron(hour, minute) {
return `${minute} ${hour} * * *`;
}
/**
* 创建自动备份任务
*/
function calculateNextRun(cronExpression) {
try {
const parts = cronExpression.split(' ');
const minute = parseInt(parts[0]) || 0;
const hour = parseInt(parts[1]) || 0;
const now = new Date();
const next = new Date(now);
next.setHours(hour, minute, 0, 0);
if (next <= now) {
next.setDate(next.getDate() + 1);
}
return next.toLocaleString('zh-CN');
} catch (error) {
return '计算失败';
}
}
function getFileSize(filePath) {
try {
if (fs.existsSync(filePath)) {
const stats = fs.statSync(filePath);
return stats.size;
}
} catch (error) {
console.error('获取文件大小失败:', error);
}
return null;
}
function createAutoBackupTask(settings) {
const { cronExpression, description, backupType, includeFiles, compress, maxCount, maxAgeDays } = settings;
@@ -84,18 +95,44 @@ function createAutoBackupTask(settings) {
throw new Error('无效的 Cron 表达式');
}
// 如果已有调度器，先停止
console.log('=== 创建自动备份任务 ===');
console.log('Cron表达式:', cronExpression);
console.log('备份类型:', backupType);
console.log('下次执行:', calculateNextRun(cronExpression));
if (schedulers.has('auto-backup')) {
console.log('停止已存在的调度器...');
stopAutoBackup();
}
// 创建新的调度器
const task = cron.schedule(cronExpression, async () => {
console.log('=== 开始执行自动备份 ===');
console.log('创建新调度器...');
const task = cron.schedule(cronExpression, async function() {
console.log('');
console.log('============================================');
console.log('=== 自动备份任务触发 ===');
console.log('触发时间:', new Date().toLocaleString('zh-CN'));
console.log('============================================');
let logId = null;
try {
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').substring(0, 19);
// 根据备份类型选择函数
console.log('创建备份日志...');
const log = await createLogEntry({
logType: 'auto',
description: `${description} - ${timestamp}`,
backupType: backupType,
includeFiles: includeFiles,
compressed: compress
});
logId = log ? log.id : null;
if (logId) {
await updateLogStatus(logId, 'running');
}
console.log('准备执行备份...');
const backupFunction = backupType === 'incremental' ? createIncrementalBackup : createBackup;
const result = await backupFunction({
@@ -109,33 +146,53 @@ function createAutoBackupTask(settings) {
if (result) {
console.log('自动备份完成:', result.filename);
console.log(`备份类型：${result.isIncremental ? '增量备份' : '全量备份'}`);
console.log('备份类型:', result.isIncremental ? '增量备份' : '全量备份');
// 上传到远端
await uploadToRemoteTargets(result.path, result.filename);
const fileSize = getFileSize(result.path);
console.log('========================\n');
const uploadResults = await uploadToRemoteTargets(result.path, result.filename);
if (logId) {
await updateLogStatus(logId, 'success', {
filename: result.filename,
filePath: result.path,
fileSize: fileSize,
remoteUploads: uploadResults
});
}
console.log('============================================\n');
} else {
console.log('无数据变化，跳过备份');
if (logId) {
await updateLogStatus(logId, 'success', {
errorMessage: '无数据变化，跳过备份'
});
}
console.log('============================================\n');
}
} catch (error) {
console.error('自动备份失败:', error);
console.error('========================\n');
console.error('错误堆栈:', error.stack);
if (logId) {
await updateLogStatus(logId, 'failed', {
errorMessage: error.message || '未知错误'
});
}
console.error('============================================\n');
}
}, {
scheduled: true,
timezone: 'Asia/Shanghai', // 设置时区为中国时区
timezone: 'Asia/Shanghai'
});
schedulers.set('auto-backup', task);
console.log(`自动备份任务已启动，Cron 表达式：${cronExpression}, 备份类型：${backupType}`);
console.log('自动备份任务已成功创建并启动');
return task;
}
/**
* 启动自动备份
*/
function startAutoBackup(settings = null) {
if (!settings) {
settings = loadSettings();
@@ -155,9 +212,6 @@ function startAutoBackup(settings = null) {
}
}
/**
* 停止自动备份
*/
function stopAutoBackup() {
if (schedulers.has('auto-backup')) {
const task = schedulers.get('auto-backup');
@@ -169,17 +223,12 @@ function stopAutoBackup() {
return false;
}
/**
* 获取自动备份状态
*/
function getAutoBackupStatus() {
const settings = loadSettings();
const isActive = schedulers.has('auto-backup');
// 计算下次执行时间
let nextRun = null;
if (isActive && settings.enabled) {
// 简单计算下次执行时间（基于当前时间和 Cron 表达式）
const now = new Date();
const [minute, hour] = settings.cronExpression.split(' ').slice(0, 2);
@@ -207,28 +256,21 @@ function getAutoBackupStatus() {
};
}
/**
* 更新自动备份设置
*/
function updateAutoBackupSettings(newSettings) {
const currentSettings = loadSettings();
const updatedSettings = { ...currentSettings, ...newSettings };
// 如果提供了小时和分钟，转换为 Cron 表达式
if (newSettings.hour !== undefined && newSettings.minute !== undefined) {
updatedSettings.cronExpression = timeToCron(newSettings.hour, newSettings.minute);
delete updatedSettings.hour;
delete updatedSettings.minute;
}
// 验证 Cron 表达式
if (!validateCronExpression(updatedSettings.cronExpression)) {
throw new Error('无效的 Cron 表达式');
}
// 保存设置
if (saveSettings(updatedSettings)) {
// 如果启用了自动备份，重新启动调度器
if (updatedSettings.enabled) {
startAutoBackup(updatedSettings);
} else {
@@ -240,9 +282,6 @@ function updateAutoBackupSettings(newSettings) {
return false;
}
/**
* 上传备份到所有启用的远端目标
*/
async function uploadToRemoteTargets(localFilePath, filename) {
const globalSettings = getGlobalSettings();
@@ -262,9 +301,9 @@ async function uploadToRemoteTargets(localFilePath, filename) {
for (const target of enabledTargets) {
try {
console.log(`开始上传到目标：${target.name} (${target.protocol})`);
console.log('开始上传到目标：' + target.name + ' (' + target.protocol + ')');
const remotePath = `${target.prefix || 'backups/'}${filename}`;
const remotePath = (target.prefix || 'backups/') + filename;
const result = await uploadToRemote(target, localFilePath, remotePath);
@@ -276,9 +315,9 @@ async function uploadToRemoteTargets(localFilePath, filename) {
...result,
});
console.log(`上传到 ${target.name} 成功`);
console.log('上传到 ' + target.name + ' 成功');
} catch (error) {
console.error(`上传到 ${target.name} 失败:`, error.message);
console.error('上传到 ' + target.name + ' 失败:', error.message);
uploadResults.push({
targetId: target.id,
targetName: target.name,
@@ -289,7 +328,6 @@ async function uploadToRemoteTargets(localFilePath, filename) {
}
}
// 检查是否需要删除本地文件
const settings = getGlobalSettings();
if (settings.deleteLocalAfterUpload && uploadResults.every(r => r.success)) {
try {
@@ -303,14 +341,34 @@ async function uploadToRemoteTargets(localFilePath, filename) {
return uploadResults;
}
/**
* 立即执行一次备份
*/
async function executeBackupNow(options = {}) {
console.log('');
console.log('============================================');
console.log('=== 手动触发备份 ===');
console.log('============================================');
let logId = null;
try {
const settings = loadSettings();
const result = await createBackup({
const backupType = options.backupType || settings.backupType || 'full';
console.log('创建备份日志...');
const log = await createLogEntry({
logType: 'manual',
description: options.description || '手动备份',
backupType: backupType,
includeFiles: options.includeFiles !== undefined ? options.includeFiles : settings.includeFiles,
compressed: options.compress !== undefined ? options.compress : settings.compress
});
logId = log ? log.id : null;
if (logId) {
await updateLogStatus(logId, 'running');
}
const backupFunction = backupType === 'incremental' ? createIncrementalBackup : createBackup;
const result = await backupFunction({
description: options.description || '手动备份',
includeFiles: options.includeFiles !== undefined ? options.includeFiles : settings.includeFiles,
compress: options.compress !== undefined ? options.compress : settings.compress,
@@ -321,10 +379,20 @@ async function executeBackupNow(options = {}) {
console.log('手动备份完成:', result.filename);
// 上传到远端
const fileSize = getFileSize(result.path);
const uploadResults = await uploadToRemoteTargets(result.path, result.filename);
console.log('====================\n');
if (logId) {
await updateLogStatus(logId, 'success', {
filename: result.filename,
filePath: result.path,
fileSize: fileSize,
remoteUploads: uploadResults
});
}
console.log('============================================\n');
return {
success: true,
result,
@@ -332,24 +400,38 @@ async function executeBackupNow(options = {}) {
};
} catch (error) {
console.error('手动备份失败:', error);
console.error('====================\n');
console.error('============================================\n');
if (logId) {
await updateLogStatus(logId, 'failed', {
errorMessage: error.message || '未知错误'
});
}
return { success: false, error: error.message };
}
}
/**
* 初始化自动备份（服务器启动时调用）
*/
function initAutoBackup() {
console.log('初始化自动备份...');
console.log('');
console.log('============================================');
console.log('=== 初始化自动备份调度器 ===');
console.log('============================================');
const settings = loadSettings();
console.log('当前设置:');
console.log(' 启用:', settings.enabled ? '是' : '否');
console.log(' Cron表达式:', settings.cronExpression);
console.log(' 备份类型:', settings.backupType);
if (settings.enabled) {
startAutoBackup(settings);
} else {
console.log('自动备份当前为禁用状态');
}
console.log('============================================\n');
return getAutoBackupStatus();
}
@@ -365,3 +447,4 @@ module.exports = {
executeBackupNow,
initAutoBackup,
};
backend/utils/backup.js
+165 -26
View File
@@ -225,11 +225,15 @@ function ensureBackupDir(backupPath) {
}
}
async function collectAllData() {
async function collectAllData(tableNames = null) {
const data = {};
let totalRecords = 0;
for (const config of BACKUP_MODELS_CONFIG) {
const configsToProcess = tableNames
? BACKUP_MODELS_CONFIG.filter(c => tableNames.includes(c.name))
: BACKUP_MODELS_CONFIG;
for (const config of configsToProcess) {
try {
const Model = require(config.modelPath);
const records = await Model.findAll({ raw: true });
@@ -545,7 +549,10 @@ async function validateBackupFile(filePath, options = {}) {
return { valid: false, error: '备份文件缺少版本信息' };
}
if (!backupData.data) {
const isIncremental = backupData.backupType === 'incremental';
const dataToValidate = isIncremental ? backupData.fullData : backupData.data;
if (!dataToValidate && !isIncremental) {
return { valid: false, error: '备份文件缺少数据内容' };
}
@@ -559,22 +566,57 @@ async function validateBackupFile(filePath, options = {}) {
}
}
// 详细的表信息统计
const tableDetails = {};
let totalRecords = 0;
for (const tableName of Object.keys(backupData.data)) {
if (Array.isArray(backupData.data[tableName])) {
const recordCount = backupData.data[tableName].length;
tableDetails[tableName] = {
recordCount,
hasData: recordCount > 0,
displayName: TABLE_NAME_MAPPING[tableName] || tableName, // 使用中文显示名称
};
totalRecords += recordCount;
if (isIncremental) {
if (backupData.fullData) {
for (const tableName of Object.keys(backupData.fullData)) {
if (Array.isArray(backupData.fullData[tableName])) {
const recordCount = backupData.fullData[tableName].length;
tableDetails[tableName] = {
recordCount,
hasData: recordCount > 0,
displayName: TABLE_NAME_MAPPING[tableName] || tableName,
type: 'full',
};
totalRecords += recordCount;
}
}
}
if (backupData.incrementalData) {
for (const tableName of Object.keys(backupData.incrementalData)) {
const inc = backupData.incrementalData[tableName];
const newCount = inc.new?.length || 0;
const updatedCount = inc.updated?.length || 0;
if (newCount > 0 || updatedCount > 0) {
tableDetails[tableName] = {
...tableDetails[tableName],
newCount,
updatedCount,
recordCount: (tableDetails[tableName]?.recordCount || 0) + newCount + updatedCount,
hasData: true,
displayName: TABLE_NAME_MAPPING[tableName] || tableName,
type: tableDetails[tableName] ? 'both' : 'incremental',
};
totalRecords += newCount + updatedCount;
}
}
}
} else {
for (const tableName of Object.keys(backupData.data)) {
if (Array.isArray(backupData.data[tableName])) {
const recordCount = backupData.data[tableName].length;
tableDetails[tableName] = {
recordCount,
hasData: recordCount > 0,
displayName: TABLE_NAME_MAPPING[tableName] || tableName,
};
totalRecords += recordCount;
}
}
}
// 文件详情
const fileDetails = {
avatars: backupData.files?.avatars?.length || 0,
others: backupData.files?.others?.length || 0,
@@ -589,6 +631,19 @@ async function validateBackupFile(filePath, options = {}) {
})) || [],
};
const metadata = isIncremental ? {
tableCount: Object.keys(backupData.fullData || {}).length,
incrementalTableCount: Object.keys(backupData.incrementalData || {}).length,
totalRecords,
totalChangedRecords: backupData.metadata?.totalChangedRecords || totalRecords,
fileCount: fileDetails.total,
lastBackupTime: backupData.lastBackupTime,
} : {
tableCount: Object.keys(backupData.data).length,
totalRecords,
fileCount: fileDetails.total,
};
return {
valid: true,
version: backupData.version,
@@ -597,12 +652,7 @@ async function validateBackupFile(filePath, options = {}) {
description: backupData.description,
compressed: backupData.compressed,
systemInfo: backupData.systemInfo,
metadata: {
tableCount: Object.keys(backupData.data).length,
totalRecords,
fileCount: fileDetails.total,
},
// 详细信息
metadata,
details: {
tables: tableDetails,
files: fileDetails,
@@ -623,9 +673,17 @@ async function restoreData(backupData, options = {}) {
recordsRestored: 0,
errors: [],
skipped: [],
tableDetails: {}, // 每个表的详细恢复信息
tableDetails: {},
};
const isIncremental = backupData.backupType === 'incremental';
const dataToRestore = isIncremental ? backupData.fullData : backupData.data;
if (!dataToRestore) {
results.errors.push({ error: '备份数据为空' });
return results;
}
for (const tableName of RESTORE_ORDER) {
if (skipTables.includes(tableName)) {
results.skipped.push(tableName);
@@ -633,7 +691,7 @@ async function restoreData(backupData, options = {}) {
continue;
}
const tableData = backupData.data[tableName];
const tableData = dataToRestore[tableName];
if (!tableData || !Array.isArray(tableData) || tableData.length === 0) {
onProgress(tableName, 'empty');
continue;
@@ -652,11 +710,9 @@ async function restoreData(backupData, options = {}) {
await Model.destroy({ where: {}, truncate: true });
}
// 预处理记录：修复 JSON 字段格式
const processedRecords = tableData.map(record => {
const processed = { ...record };
// 修复 Device 表的 customFields 字段
if (tableName === 'Device' && processed.customFields !== undefined && processed.customFields !== null) {
if (typeof processed.customFields === 'string') {
try {
@@ -701,7 +757,6 @@ async function restoreData(backupData, options = {}) {
results.tablesRestored++;
results.recordsRestored += insertedCount;
// 记录每个表的详细信息
results.tableDetails[tableName] = {
recordCount: insertedCount,
displayName: TABLE_NAME_MAPPING[tableName] || tableName,
@@ -715,6 +770,89 @@ async function restoreData(backupData, options = {}) {
}
}
if (isIncremental && backupData.incrementalData) {
console.log('\n恢复增量数据...');
const incrementalResults = await restoreIncrementalData(backupData.incrementalData, options);
results.tablesRestored += incrementalResults.tablesRestored;
results.recordsRestored += incrementalResults.recordsRestored;
results.errors.push(...incrementalResults.errors);
Object.assign(results.tableDetails, incrementalResults.tableDetails);
}
return results;
}
async function restoreIncrementalData(incrementalData, options = {}) {
const results = {
tablesRestored: 0,
recordsRestored: 0,
errors: [],
tableDetails: {},
};
for (const tableName of Object.keys(incrementalData)) {
const tableIncrement = incrementalData[tableName];
const config = BACKUP_MODELS_CONFIG.find(c => c.name === tableName);
if (!config) {
results.errors.push({ table: tableName, error: '未找到模型配置' });
continue;
}
try {
const Model = require(config.modelPath);
let updatedCount = 0;
if (tableIncrement.new && tableIncrement.new.length > 0) {
for (const record of tableIncrement.new) {
try {
await Model.create(record, { validate: false, silent: true });
updatedCount++;
} catch (insertError) {
if (insertError.name === 'SequelizeUniqueConstraintError') {
await Model.upsert(record, { validate: false, silent: true });
updatedCount++;
} else {
results.errors.push({
table: tableName,
record: record[Object.keys(record)[0]],
error: `新增失败: ${insertError.message}`,
});
}
}
}
}
if (tableIncrement.updated && tableIncrement.updated.length > 0) {
for (const record of tableIncrement.updated) {
try {
await Model.upsert(record, { validate: false, silent: true });
updatedCount++;
} catch (updateError) {
results.errors.push({
table: tableName,
record: record[Object.keys(record)[0]],
error: `更新失败: ${updateError.message}`,
});
}
}
}
if (updatedCount > 0) {
results.tablesRestored++;
results.recordsRestored += updatedCount;
results.tableDetails[tableName] = {
recordCount: updatedCount,
displayName: TABLE_NAME_MAPPING[tableName] || tableName,
success: true,
isIncremental: true,
};
}
} catch (error) {
results.errors.push({ table: tableName, error: error.message });
}
}
return results;
}
@@ -855,7 +993,7 @@ function cleanOldBackups(options = {}) {
const maxAgeMs = maxAgeDays * 24 * 60 * 60 * 1000;
const files = fs.readdirSync(backupPath)
.filter(f => (f.startsWith('backup_') || f.startsWith('uploaded_')) && (f.endsWith('.json') || f.endsWith('.json.gz')))
.filter(f => (f.startsWith('backup_') || f.startsWith('uploaded_') || f.startsWith('incremental_')) && (f.endsWith('.json') || f.endsWith('.json.gz')))
.map(f => {
const filePath = path.join(backupPath, f);
const stats = fs.statSync(filePath);
@@ -928,6 +1066,7 @@ module.exports = {
getLastBackupTime,
validateBackupFile,
restoreData,
restoreIncrementalData,
restoreFiles,
restoreBackup,
calculateChecksum,
backend/utils/backupLog.js
+125
View File
@@ -0,0 +1,125 @@
const BackupLog = require('../models/BackupLog');
const fs = require('fs');
async function createLogEntry(options) {
const { logType, description, backupType, includeFiles, compressed } = options;
try {
const log = await BackupLog.create({
logType: logType || 'manual',
status: 'pending',
description: description || '',
backupType: backupType || 'full',
includeFiles: includeFiles || false,
compressed: compressed || false,
startTime: new Date()
});
return log;
} catch (error) {
console.error('创建备份日志失败:', error);
return null;
}
}
async function updateLogStatus(logId, status, options = {}) {
try {
const updateData = { status };
if (status === 'running') {
updateData.startTime = new Date();
}
if (status === 'success' || status === 'failed') {
updateData.endTime = new Date();
const log = await BackupLog.findByPk(logId);
if (log && log.startTime) {
updateData.duration = new Date() - new Date(log.startTime);
}
}
if (options.filename) updateData.filename = options.filename;
if (options.filePath) updateData.filePath = options.filePath;
if (options.fileSize) updateData.fileSize = options.fileSize;
if (options.errorMessage) updateData.errorMessage = options.errorMessage;
if (options.remoteUploads) updateData.remoteUploads = options.remoteUploads;
await BackupLog.update(updateData, {
where: { id: logId }
});
return true;
} catch (error) {
console.error('更新备份日志失败:', error);
return false;
}
}
async function getBackupLogs(options = {}) {
try {
const { page = 1, pageSize = 20, logType, status } = options;
const where = {};
if (logType) where.logType = logType;
if (status) where.status = status;
const offset = (page - 1) * pageSize;
const { count, rows } = await BackupLog.findAndCountAll({
where,
order: [['createdAt', 'DESC']],
limit: pageSize,
offset
});
return {
logs: rows,
total: count,
page,
pageSize,
totalPages: Math.ceil(count / pageSize)
};
} catch (error) {
console.error('获取备份日志失败:', error);
return { logs: [], total: 0, page: 1, pageSize: 20, totalPages: 0 };
}
}
async function getBackupLogById(id) {
try {
return await BackupLog.findByPk(id);
} catch (error) {
console.error('获取备份日志详情失败:', error);
return null;
}
}
async function deleteOldLogs(days = 30) {
try {
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - days);
const deletedCount = await BackupLog.destroy({
where: {
createdAt: {
[require('sequelize').Op.lt]: cutoffDate
}
}
});
console.log(`删除了 ${deletedCount} 条旧备份日志`);
return deletedCount;
} catch (error) {
console.error('删除旧备份日志失败:', error);
return 0;
}
}
module.exports = {
createLogEntry,
updateLogStatus,
getBackupLogs,
getBackupLogById,
deleteOldLogs
};