mirror of
https://github.com/khoaliber/dockhand.git
synced 2026-03-03 21:19:06 +00:00
510 lines
17 KiB
TypeScript
510 lines
17 KiB
TypeScript
/**
|
|
* Environment Update Check Task
|
|
*
|
|
* Checks all containers in an environment for available image updates.
|
|
* Can optionally auto-update containers when updates are found.
|
|
*/
|
|
|
|
import type { ScheduleTrigger, VulnerabilityCriteria } from '../../db';
|
|
import {
|
|
getEnvUpdateCheckSettings,
|
|
getEnvironment,
|
|
createScheduleExecution,
|
|
updateScheduleExecution,
|
|
appendScheduleExecutionLog,
|
|
saveVulnerabilityScan,
|
|
clearPendingContainerUpdates,
|
|
addPendingContainerUpdate,
|
|
removePendingContainerUpdate
|
|
} from '../../db';
|
|
import {
|
|
listContainers,
|
|
inspectContainer,
|
|
checkImageUpdateAvailable,
|
|
pullImage,
|
|
stopContainer,
|
|
removeContainer,
|
|
createContainer,
|
|
getTempImageTag,
|
|
isDigestBasedImage,
|
|
getImageIdByTag,
|
|
removeTempImage,
|
|
tagImage
|
|
} from '../../docker';
|
|
import { sendEventNotification } from '../../notifications';
|
|
import { getScannerSettings, scanImage, type VulnerabilitySeverity } from '../../scanner';
|
|
import { parseImageNameAndTag, shouldBlockUpdate, combineScanSummaries, isDockhandContainer } from './update-utils';
|
|
|
|
interface UpdateInfo {
|
|
containerId: string;
|
|
containerName: string;
|
|
imageName: string;
|
|
currentImageId: string;
|
|
currentDigest?: string;
|
|
newDigest?: string;
|
|
}
|
|
|
|
// Track running update checks to prevent concurrent execution
|
|
const runningUpdateChecks = new Set<number>();
|
|
|
|
/**
|
|
* Execute environment update check job.
|
|
* @param environmentId - The environment ID to check
|
|
* @param triggeredBy - What triggered this execution
|
|
*/
|
|
export async function runEnvUpdateCheckJob(
|
|
environmentId: number,
|
|
triggeredBy: ScheduleTrigger = 'cron'
|
|
): Promise<void> {
|
|
// Prevent concurrent execution for the same environment
|
|
if (runningUpdateChecks.has(environmentId)) {
|
|
console.log(`[EnvUpdateCheck] Environment ${environmentId} update check already running, skipping`);
|
|
return;
|
|
}
|
|
|
|
runningUpdateChecks.add(environmentId);
|
|
const startTime = Date.now();
|
|
|
|
try {
|
|
// Get environment info
|
|
const env = await getEnvironment(environmentId);
|
|
if (!env) {
|
|
console.error(`[EnvUpdateCheck] Environment ${environmentId} not found`);
|
|
return;
|
|
}
|
|
|
|
// Get settings
|
|
const config = await getEnvUpdateCheckSettings(environmentId);
|
|
if (!config) {
|
|
console.error(`[EnvUpdateCheck] No settings found for environment ${environmentId}`);
|
|
return;
|
|
}
|
|
|
|
// Create execution record
|
|
const execution = await createScheduleExecution({
|
|
scheduleType: 'env_update_check',
|
|
scheduleId: environmentId,
|
|
environmentId,
|
|
entityName: `Update: ${env.name}`,
|
|
triggeredBy,
|
|
status: 'running'
|
|
});
|
|
|
|
await updateScheduleExecution(execution.id, {
|
|
startedAt: new Date().toISOString()
|
|
});
|
|
|
|
const log = async (message: string) => {
|
|
console.log(`[EnvUpdateCheck] ${message}`);
|
|
await appendScheduleExecutionLog(execution.id, `[${new Date().toISOString()}] ${message}`);
|
|
};
|
|
|
|
try {
|
|
await log(`Starting update check for environment: ${env.name}`);
|
|
await log(`Auto-update mode: ${config.autoUpdate ? 'ON' : 'OFF'}`);
|
|
|
|
// Clear pending updates at the start - we'll re-add as we discover updates
|
|
await clearPendingContainerUpdates(environmentId);
|
|
|
|
// Get all containers in this environment
|
|
const containers = await listContainers(true, environmentId);
|
|
await log(`Found ${containers.length} containers`);
|
|
|
|
const updatesAvailable: UpdateInfo[] = [];
|
|
let checkedCount = 0;
|
|
let errorCount = 0;
|
|
|
|
// Check each container for updates
|
|
for (const container of containers) {
|
|
try {
|
|
const inspectData = await inspectContainer(container.id, environmentId) as any;
|
|
const imageName = inspectData.Config?.Image;
|
|
const currentImageId = inspectData.Image;
|
|
|
|
if (!imageName) {
|
|
await log(` [${container.name}] Skipping - no image name found`);
|
|
continue;
|
|
}
|
|
|
|
checkedCount++;
|
|
await log(` Checking: ${container.name} (${imageName})`);
|
|
|
|
const result = await checkImageUpdateAvailable(imageName, currentImageId, environmentId);
|
|
|
|
if (result.isLocalImage) {
|
|
await log(` Local image - skipping update check`);
|
|
continue;
|
|
}
|
|
|
|
if (result.error) {
|
|
await log(` Error: ${result.error}`);
|
|
errorCount++;
|
|
continue;
|
|
}
|
|
|
|
if (result.hasUpdate) {
|
|
updatesAvailable.push({
|
|
containerId: container.id,
|
|
containerName: container.name,
|
|
imageName,
|
|
currentImageId,
|
|
currentDigest: result.currentDigest,
|
|
newDigest: result.registryDigest
|
|
});
|
|
// Add to pending table immediately - will be removed on successful update
|
|
await addPendingContainerUpdate(environmentId, container.id, container.name, imageName);
|
|
await log(` UPDATE AVAILABLE`);
|
|
await log(` Current: ${result.currentDigest?.substring(0, 24) || 'unknown'}...`);
|
|
await log(` New: ${result.registryDigest?.substring(0, 24) || 'unknown'}...`);
|
|
} else {
|
|
await log(` Up to date`);
|
|
}
|
|
} catch (err: any) {
|
|
await log(` [${container.name}] Error: ${err.message}`);
|
|
errorCount++;
|
|
}
|
|
}
|
|
|
|
// Summary
|
|
await log('');
|
|
await log('=== SUMMARY ===');
|
|
await log(`Total containers: ${containers.length}`);
|
|
await log(`Checked: ${checkedCount}`);
|
|
await log(`Updates available: ${updatesAvailable.length}`);
|
|
await log(`Errors: ${errorCount}`);
|
|
|
|
if (updatesAvailable.length === 0) {
|
|
await log('All containers are up to date');
|
|
// Pending updates already cleared at start, nothing to add
|
|
await updateScheduleExecution(execution.id, {
|
|
status: 'success',
|
|
completedAt: new Date().toISOString(),
|
|
duration: Date.now() - startTime,
|
|
details: {
|
|
updatesFound: 0,
|
|
containersChecked: checkedCount,
|
|
errors: errorCount
|
|
}
|
|
});
|
|
return;
|
|
}
|
|
|
|
// Build notification message with details
|
|
const updateList = updatesAvailable
|
|
.map(u => {
|
|
const currentShort = u.currentDigest?.substring(0, 12) || 'unknown';
|
|
const newShort = u.newDigest?.substring(0, 12) || 'unknown';
|
|
return `- ${u.containerName} (${u.imageName})\n ${currentShort}... -> ${newShort}...`;
|
|
})
|
|
.join('\n');
|
|
|
|
if (config.autoUpdate) {
|
|
// Auto-update mode: actually update the containers with safe-pull flow
|
|
await log('');
|
|
await log('=== AUTO-UPDATE MODE ===');
|
|
|
|
// Get scanner settings and vulnerability criteria
|
|
const scannerSettings = await getScannerSettings(environmentId);
|
|
const vulnerabilityCriteria = (config.vulnerabilityCriteria || 'never') as VulnerabilityCriteria;
|
|
// Scan if scanning is enabled (scanner !== 'none')
|
|
// The vulnerabilityCriteria only controls whether to BLOCK updates, not whether to SCAN
|
|
const shouldScan = scannerSettings.scanner !== 'none';
|
|
|
|
await log(`Vulnerability criteria: ${vulnerabilityCriteria}`);
|
|
if (shouldScan) {
|
|
await log(`Scanner: ${scannerSettings.scanner} (scan enabled)`);
|
|
}
|
|
await log(`Updating ${updatesAvailable.length} containers...`);
|
|
|
|
let successCount = 0;
|
|
let failCount = 0;
|
|
let blockedCount = 0;
|
|
const updatedContainers: string[] = [];
|
|
const failedContainers: string[] = [];
|
|
const blockedContainers: { name: string; reason: string; scannerResults?: { scanner: string; critical: number; high: number; medium: number; low: number }[] }[] = [];
|
|
|
|
for (const update of updatesAvailable) {
|
|
// Skip Dockhand container - cannot update itself
|
|
if (isDockhandContainer(update.imageName)) {
|
|
await log(`\n[${update.containerName}] Skipping - cannot auto-update Dockhand itself`);
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
await log(`\nUpdating: ${update.containerName}`);
|
|
|
|
// Get full container config
|
|
const inspectData = await inspectContainer(update.containerId, environmentId) as any;
|
|
const wasRunning = inspectData.State.Running;
|
|
const containerConfig = inspectData.Config;
|
|
const hostConfig = inspectData.HostConfig;
|
|
|
|
// SAFE-PULL FLOW
|
|
if (shouldScan && !isDigestBasedImage(update.imageName)) {
|
|
const tempTag = getTempImageTag(update.imageName);
|
|
await log(` Safe-pull with temp tag: ${tempTag}`);
|
|
|
|
// Step 1: Pull new image
|
|
await log(` Pulling ${update.imageName}...`);
|
|
await pullImage(update.imageName, () => {}, environmentId);
|
|
|
|
// Step 2: Get new image ID
|
|
const newImageId = await getImageIdByTag(update.imageName, environmentId);
|
|
if (!newImageId) {
|
|
throw new Error('Failed to get new image ID after pull');
|
|
}
|
|
await log(` New image: ${newImageId.substring(0, 19)}`);
|
|
|
|
// Step 3: SAFETY - Restore original tag to old image
|
|
const [oldRepo, oldTag] = parseImageNameAndTag(update.imageName);
|
|
await tagImage(update.currentImageId, oldRepo, oldTag, environmentId);
|
|
await log(` Restored original tag to safe image`);
|
|
|
|
// Step 4: Tag new image with temp suffix
|
|
const [tempRepo, tempTagName] = parseImageNameAndTag(tempTag);
|
|
await tagImage(newImageId, tempRepo, tempTagName, environmentId);
|
|
|
|
// Step 5: Scan temp image
|
|
await log(` Scanning for vulnerabilities...`);
|
|
let scanBlocked = false;
|
|
let blockReason = '';
|
|
let currentScannerResults: { scanner: string; critical: number; high: number; medium: number; low: number }[] = [];
|
|
|
|
// Collect scan logs to log after scan completes
|
|
const scanLogs: string[] = [];
|
|
|
|
try {
|
|
const scanResults = await scanImage(tempTag, environmentId, (progress) => {
|
|
if (progress.message) {
|
|
scanLogs.push(` [${progress.scanner || 'scan'}] ${progress.message}`);
|
|
}
|
|
});
|
|
|
|
// Log collected scan messages
|
|
for (const scanLog of scanLogs) {
|
|
await log(scanLog);
|
|
}
|
|
|
|
if (scanResults.length > 0) {
|
|
const scanSummary = combineScanSummaries(scanResults);
|
|
await log(` Scan: ${scanSummary.critical} critical, ${scanSummary.high} high, ${scanSummary.medium} medium, ${scanSummary.low} low`);
|
|
|
|
// Capture per-scanner results for blocking info
|
|
currentScannerResults = scanResults.map(r => ({
|
|
scanner: r.scanner,
|
|
critical: r.summary.critical,
|
|
high: r.summary.high,
|
|
medium: r.summary.medium,
|
|
low: r.summary.low
|
|
}));
|
|
|
|
// Save scan results
|
|
for (const result of scanResults) {
|
|
try {
|
|
await saveVulnerabilityScan({
|
|
environmentId,
|
|
imageId: newImageId,
|
|
imageName: result.imageName,
|
|
scanner: result.scanner,
|
|
scannedAt: result.scannedAt,
|
|
scanDuration: result.scanDuration,
|
|
criticalCount: result.summary.critical,
|
|
highCount: result.summary.high,
|
|
mediumCount: result.summary.medium,
|
|
lowCount: result.summary.low,
|
|
negligibleCount: result.summary.negligible,
|
|
unknownCount: result.summary.unknown,
|
|
vulnerabilities: result.vulnerabilities,
|
|
error: result.error ?? null
|
|
});
|
|
} catch { /* ignore save errors */ }
|
|
}
|
|
|
|
// Check if blocked
|
|
const { blocked, reason } = shouldBlockUpdate(vulnerabilityCriteria, scanSummary, undefined);
|
|
if (blocked) {
|
|
scanBlocked = true;
|
|
blockReason = reason;
|
|
}
|
|
}
|
|
} catch (scanErr: any) {
|
|
await log(` Scan failed: ${scanErr.message}`);
|
|
scanBlocked = true;
|
|
blockReason = `Scan failed: ${scanErr.message}`;
|
|
}
|
|
|
|
if (scanBlocked) {
|
|
// BLOCKED - Remove temp image
|
|
await log(` UPDATE BLOCKED: ${blockReason}`);
|
|
await removeTempImage(newImageId, environmentId);
|
|
await log(` Removed blocked image - container stays safe`);
|
|
blockedCount++;
|
|
blockedContainers.push({
|
|
name: update.containerName,
|
|
reason: blockReason,
|
|
scannerResults: currentScannerResults.length > 0 ? currentScannerResults : undefined
|
|
});
|
|
continue;
|
|
}
|
|
|
|
// APPROVED - Re-tag to original
|
|
await log(` Scan passed, re-tagging...`);
|
|
await tagImage(newImageId, oldRepo, oldTag, environmentId);
|
|
try {
|
|
await removeTempImage(tempTag, environmentId);
|
|
} catch { /* ignore cleanup errors */ }
|
|
} else {
|
|
// Simple pull (no scanning or digest-based image)
|
|
await log(` Pulling ${update.imageName}...`);
|
|
await pullImage(update.imageName, () => {}, environmentId);
|
|
}
|
|
|
|
// Stop container if running
|
|
if (wasRunning) {
|
|
await log(` Stopping...`);
|
|
await stopContainer(update.containerId, environmentId);
|
|
}
|
|
|
|
// Remove old container
|
|
await log(` Removing old container...`);
|
|
await removeContainer(update.containerId, true, environmentId);
|
|
|
|
// Prepare port bindings
|
|
const ports: { [key: string]: { HostPort: string } } = {};
|
|
if (hostConfig.PortBindings) {
|
|
for (const [containerPort, bindings] of Object.entries(hostConfig.PortBindings)) {
|
|
if (bindings && (bindings as any[]).length > 0) {
|
|
ports[containerPort] = { HostPort: (bindings as any[])[0].HostPort || '' };
|
|
}
|
|
}
|
|
}
|
|
|
|
// Create new container
|
|
await log(` Creating new container...`);
|
|
const newContainer = await createContainer({
|
|
name: update.containerName,
|
|
image: update.imageName,
|
|
ports,
|
|
volumeBinds: hostConfig.Binds || [],
|
|
env: containerConfig.Env || [],
|
|
labels: containerConfig.Labels || {},
|
|
cmd: containerConfig.Cmd || undefined,
|
|
restartPolicy: hostConfig.RestartPolicy?.Name || 'no',
|
|
networkMode: hostConfig.NetworkMode || undefined
|
|
}, environmentId);
|
|
|
|
// Start if was running
|
|
if (wasRunning) {
|
|
await log(` Starting...`);
|
|
await newContainer.start();
|
|
}
|
|
|
|
await log(` Updated successfully`);
|
|
successCount++;
|
|
updatedContainers.push(update.containerName);
|
|
// Remove from pending table - successfully updated
|
|
await removePendingContainerUpdate(environmentId, update.containerId);
|
|
} catch (err: any) {
|
|
await log(` FAILED: ${err.message}`);
|
|
failCount++;
|
|
failedContainers.push(update.containerName);
|
|
}
|
|
}
|
|
|
|
await log('');
|
|
await log(`=== UPDATE COMPLETE ===`);
|
|
await log(`Updated: ${successCount}`);
|
|
await log(`Blocked: ${blockedCount}`);
|
|
await log(`Failed: ${failCount}`);
|
|
|
|
// Send notifications
|
|
if (blockedCount > 0) {
|
|
await sendEventNotification('auto_update_blocked', {
|
|
title: `${blockedCount} update(s) blocked in ${env.name}`,
|
|
message: blockedContainers.map(c => `- ${c.name}: ${c.reason}`).join('\n'),
|
|
type: 'warning'
|
|
}, environmentId);
|
|
}
|
|
|
|
const notificationMessage = successCount > 0
|
|
? `Updated ${successCount} container(s) in ${env.name}:\n${updatedContainers.map(c => `- ${c}`).join('\n')}${blockedCount > 0 ? `\n\nBlocked (${blockedCount}):\n${blockedContainers.map(c => `- ${c.name}`).join('\n')}` : ''}${failCount > 0 ? `\n\nFailed (${failCount}):\n${failedContainers.map(c => `- ${c}`).join('\n')}` : ''}`
|
|
: blockedCount > 0 ? `All updates blocked in ${env.name}` : `Update failed for all containers in ${env.name}`;
|
|
|
|
await sendEventNotification('batch_update_success', {
|
|
title: successCount > 0 ? `Containers updated in ${env.name}` : blockedCount > 0 ? `Updates blocked in ${env.name}` : `Container updates failed in ${env.name}`,
|
|
message: notificationMessage,
|
|
type: successCount > 0 && failCount === 0 && blockedCount === 0 ? 'success' : successCount > 0 ? 'warning' : 'error'
|
|
}, environmentId);
|
|
|
|
// Blocked/failed containers stay in pending table (successfully updated ones were removed)
|
|
|
|
await updateScheduleExecution(execution.id, {
|
|
status: failCount > 0 && successCount === 0 && blockedCount === 0 ? 'failed' : 'success',
|
|
completedAt: new Date().toISOString(),
|
|
duration: Date.now() - startTime,
|
|
details: {
|
|
mode: 'auto_update',
|
|
updatesFound: updatesAvailable.length,
|
|
containersChecked: checkedCount,
|
|
errors: errorCount,
|
|
autoUpdate: true,
|
|
vulnerabilityCriteria,
|
|
summary: { checked: checkedCount, updated: successCount, blocked: blockedCount, failed: failCount },
|
|
containers: [
|
|
...updatedContainers.map(name => ({ name, status: 'updated' as const })),
|
|
...blockedContainers.map(c => ({ name: c.name, status: 'blocked' as const, blockReason: c.reason, scannerResults: c.scannerResults })),
|
|
...failedContainers.map(name => ({ name, status: 'failed' as const }))
|
|
],
|
|
updated: successCount,
|
|
blocked: blockedCount,
|
|
failed: failCount,
|
|
blockedContainers
|
|
}
|
|
});
|
|
} else {
|
|
// Check-only mode: just send notification
|
|
await log('');
|
|
await log('Check-only mode - sending notification about available updates');
|
|
// Pending updates already added as we discovered them
|
|
|
|
await sendEventNotification('updates_detected', {
|
|
title: `Container updates available in ${env.name}`,
|
|
message: `${updatesAvailable.length} update(s) available:\n${updateList}`,
|
|
type: 'info'
|
|
}, environmentId);
|
|
|
|
await updateScheduleExecution(execution.id, {
|
|
status: 'success',
|
|
completedAt: new Date().toISOString(),
|
|
duration: Date.now() - startTime,
|
|
details: {
|
|
mode: 'notify_only',
|
|
updatesFound: updatesAvailable.length,
|
|
containersChecked: checkedCount,
|
|
errors: errorCount,
|
|
autoUpdate: false,
|
|
summary: { checked: checkedCount, updated: 0, blocked: 0, failed: 0 },
|
|
containers: updatesAvailable.map(u => ({
|
|
name: u.containerName,
|
|
status: 'checked' as const,
|
|
imageName: u.imageName,
|
|
currentDigest: u.currentDigest,
|
|
newDigest: u.newDigest
|
|
}))
|
|
}
|
|
});
|
|
}
|
|
} catch (error: any) {
|
|
await log(`Error: ${error.message}`);
|
|
await updateScheduleExecution(execution.id, {
|
|
status: 'failed',
|
|
completedAt: new Date().toISOString(),
|
|
duration: Date.now() - startTime,
|
|
errorMessage: error.message
|
|
});
|
|
}
|
|
} finally {
|
|
runningUpdateChecks.delete(environmentId);
|
|
}
|
|
}
|