import type { RequestHandler } from '@sveltejs/kit'; import { getEnvironments, getLatestHostMetrics, getHostMetrics, getContainerEventStats, getContainerEvents, getEnvSetting, getEnvUpdateCheckSettings } from '$lib/server/db'; import { listContainers, listImages, listNetworks, getDockerInfo, getContainerStats, getDiskUsage } from '$lib/server/docker'; import { listComposeStacks } from '$lib/server/stacks'; import { authorize } from '$lib/server/authorize'; import type { EnvironmentStats } from '../+server'; import { parseLabels } from '$lib/utils/label-colors'; // Helper to add timeout to promises function withTimeout(promise: Promise, ms: number, fallback: T): Promise { return Promise.race([ promise, new Promise((resolve) => setTimeout(() => resolve(fallback), ms)) ]); } // Disk usage cache - getDiskUsage() is very slow (30s timeout) but data changes rarely // Cache per environment with 5-minute TTL interface DiskUsageCache { data: any; timestamp: number; } const diskUsageCache: Map = new Map(); const DISK_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes const MAX_CACHE_SIZE = 100; // Maximum environments to cache // Cleanup expired cache entries periodically to prevent unbounded growth // Also limits cache size for environments that were deleted setInterval(() => { const now = Date.now(); // Remove expired entries for (const [envId, cached] of diskUsageCache.entries()) { if (now - cached.timestamp > DISK_CACHE_TTL_MS * 2) { diskUsageCache.delete(envId); } } // Enforce max size by removing oldest entries if (diskUsageCache.size > MAX_CACHE_SIZE) { const entries = Array.from(diskUsageCache.entries()) .sort((a, b) => a[1].timestamp - b[1].timestamp); const toRemove = entries.slice(0, entries.length - MAX_CACHE_SIZE); for (const [envId] of toRemove) { diskUsageCache.delete(envId); } } }, 10 * 60 * 1000); // Every 10 minutes async function getCachedDiskUsage(envId: number): Promise { const cached = diskUsageCache.get(envId); const now = Date.now(); // Return cached data if still valid if (cached && (now - cached.timestamp) < DISK_CACHE_TTL_MS) { return cached.data; } // Fetch fresh data with timeout const data = await withTimeout(getDiskUsage(envId).catch(() => null), 30000, null); // Only cache successful results - if fetch failed, retry on next request if (data !== null) { diskUsageCache.set(envId, { data, timestamp: now }); } return data; } // Limit for per-container stats (reduced from 15 to improve performance) const TOP_CONTAINERS_LIMIT = 8; // Calculate CPU percentage from Docker stats (same logic as container stats endpoint) function calculateCpuPercent(stats: any): number { const cpuDelta = stats.cpu_stats.cpu_usage.total_usage - stats.precpu_stats.cpu_usage.total_usage; const systemDelta = stats.cpu_stats.system_cpu_usage - stats.precpu_stats.system_cpu_usage; const cpuCount = stats.cpu_stats.online_cpus || stats.cpu_stats.cpu_usage.percpu_usage?.length || 1; if (systemDelta > 0 && cpuDelta > 0) { return (cpuDelta / systemDelta) * cpuCount * 100; } return 0; } // Progressive stats loading - returns stats object and emits partial updates via callback async function getEnvironmentStatsProgressive( env: any, onPartialUpdate: (stats: Partial & { id: number }) => void ): Promise { const envStats: EnvironmentStats = { id: env.id, name: env.name, host: env.host ?? undefined, port: env.port ?? undefined, icon: env.icon || 'globe', socketPath: env.socketPath ?? undefined, collectActivity: env.collectActivity, collectMetrics: env.collectMetrics ?? true, scannerEnabled: false, updateCheckEnabled: false, updateCheckAutoUpdate: false, labels: parseLabels(env.labels), connectionType: (env.connectionType as 'socket' | 'direct' | 'hawser-standard' | 'hawser-edge') || 'socket', online: false, containers: { total: 0, running: 0, stopped: 0, paused: 0, restarting: 0, unhealthy: 0 }, images: { total: 0, totalSize: 0 }, volumes: { total: 0, totalSize: 0 }, containersSize: 0, buildCacheSize: 0, networks: { total: 0 }, stacks: { total: 0, running: 0, partial: 0, stopped: 0 }, metrics: null, events: { total: 0, today: 0 }, topContainers: [], recentEvents: [], // Loading states for progressive display loading: { containers: true, images: true, volumes: true, networks: true, stacks: true, diskUsage: true, topContainers: true } }; try { // Check scanner settings - scanner type is stored in 'vulnerability_scanner' const scannerType = await getEnvSetting('vulnerability_scanner', env.id); envStats.scannerEnabled = scannerType && scannerType !== 'none'; // Check update check settings const updateCheckSettings = await getEnvUpdateCheckSettings(env.id); if (updateCheckSettings && updateCheckSettings.enabled) { envStats.updateCheckEnabled = true; envStats.updateCheckAutoUpdate = updateCheckSettings.autoUpdate; } // Check if Docker is accessible (with 5 second timeout) const dockerInfo = await withTimeout(getDockerInfo(env.id), 5000, null); if (!dockerInfo) { envStats.error = 'Connection timeout or Docker not accessible'; envStats.loading = undefined; // Clear loading states on error // Send offline status to client onPartialUpdate({ id: env.id, online: false, error: envStats.error, loading: undefined }); return envStats; } envStats.online = true; // Get all database stats in parallel for better performance const [latestMetrics, eventStats, recentEventsResult, metricsHistory] = await Promise.all([ getLatestHostMetrics(env.id), getContainerEventStats(env.id), getContainerEvents({ environmentId: env.id, limit: 10 }), getHostMetrics(30, env.id) ]); if (latestMetrics) { envStats.metrics = { cpuPercent: latestMetrics.cpuPercent, memoryPercent: latestMetrics.memoryPercent, memoryUsed: latestMetrics.memoryUsed, memoryTotal: latestMetrics.memoryTotal }; } envStats.events = { total: eventStats.total, today: eventStats.today }; if (recentEventsResult.events.length > 0) { envStats.recentEvents = recentEventsResult.events.map(e => ({ container_name: e.containerName || 'unknown', action: e.action, timestamp: e.timestamp })); } if (metricsHistory.length > 0) { envStats.metricsHistory = metricsHistory.reverse().map(m => ({ cpu_percent: m.cpuPercent, memory_percent: m.memoryPercent, timestamp: m.timestamp })); } // Send initial update with DB data and online status onPartialUpdate({ id: env.id, online: true, metrics: envStats.metrics, events: envStats.events, recentEvents: envStats.recentEvents, metricsHistory: envStats.metricsHistory, scannerEnabled: envStats.scannerEnabled, updateCheckEnabled: envStats.updateCheckEnabled, updateCheckAutoUpdate: envStats.updateCheckAutoUpdate, loading: { ...envStats.loading } }); // Helper to get valid size const getValidSize = (size: number | undefined | null): number => { return size && size > 0 ? size : 0; }; // PHASE 1: Containers (usually fast) const containersPromise = withTimeout(listContainers(true, env.id).catch(() => []), 10000, []) .then(async (containers) => { envStats.containers.total = containers.length; envStats.containers.running = containers.filter((c: any) => c.state === 'running').length; envStats.containers.stopped = containers.filter((c: any) => c.state === 'exited').length; envStats.containers.paused = containers.filter((c: any) => c.state === 'paused').length; envStats.containers.restarting = containers.filter((c: any) => c.state === 'restarting').length; envStats.containers.unhealthy = containers.filter((c: any) => c.health === 'unhealthy').length; envStats.loading!.containers = false; onPartialUpdate({ id: env.id, containers: { ...envStats.containers }, loading: { ...envStats.loading! } }); return containers; }); // PHASE 2: Images, Networks, Stacks (medium speed) - run in parallel const imagesPromise = withTimeout(listImages(env.id).catch(() => []), 10000, []) .then((images) => { envStats.images.total = images.length; envStats.images.totalSize = images.reduce((sum: number, img: any) => sum + getValidSize(img.size), 0); envStats.loading!.images = false; onPartialUpdate({ id: env.id, images: { ...envStats.images }, loading: { ...envStats.loading! } }); return images; }); const networksPromise = withTimeout(listNetworks(env.id).catch(() => []), 10000, []) .then((networks) => { envStats.networks.total = networks.length; envStats.loading!.networks = false; onPartialUpdate({ id: env.id, networks: { ...envStats.networks }, loading: { ...envStats.loading! } }); return networks; }); const stacksPromise = withTimeout(listComposeStacks(env.id).catch(() => []), 10000, []) .then((stacks) => { envStats.stacks.total = stacks.length; envStats.stacks.running = stacks.filter((s: any) => s.status === 'running').length; envStats.stacks.partial = stacks.filter((s: any) => s.status === 'partial').length; envStats.stacks.stopped = stacks.filter((s: any) => s.status === 'stopped').length; envStats.loading!.stacks = false; onPartialUpdate({ id: env.id, stacks: { ...envStats.stacks }, loading: { ...envStats.loading! } }); return stacks; }); // PHASE 3: Disk usage (slow - includes volumes) - uses cache for better performance const diskUsagePromise = getCachedDiskUsage(env.id) .then((diskUsage) => { if (diskUsage) { // Update images with disk usage data (more accurate) envStats.images.total = diskUsage.Images?.length || envStats.images.total; envStats.images.totalSize = diskUsage.Images?.reduce((sum: number, img: any) => sum + getValidSize(img.Size), 0) || envStats.images.totalSize; // Volumes from disk usage envStats.volumes.total = diskUsage.Volumes?.length || 0; envStats.volumes.totalSize = diskUsage.Volumes?.reduce((sum: number, vol: any) => sum + getValidSize(vol.UsageData?.Size), 0) || 0; // Containers disk size envStats.containersSize = diskUsage.Containers?.reduce((sum: number, c: any) => sum + getValidSize(c.SizeRw), 0) || 0; // Build cache envStats.buildCacheSize = diskUsage.BuildCache?.reduce((sum: number, bc: any) => sum + getValidSize(bc.Size), 0) || 0; } envStats.loading!.volumes = false; envStats.loading!.diskUsage = false; onPartialUpdate({ id: env.id, images: { ...envStats.images }, volumes: { ...envStats.volumes }, containersSize: envStats.containersSize, buildCacheSize: envStats.buildCacheSize, loading: { ...envStats.loading! } }); return diskUsage; }); // PHASE 4: Top containers (slow - requires per-container stats) // Limited to TOP_CONTAINERS_LIMIT containers to reduce API calls const topContainersPromise = containersPromise.then(async (containers) => { const runningContainersList = containers.filter((c: any) => c.state === 'running'); const topContainersPromises = runningContainersList.slice(0, TOP_CONTAINERS_LIMIT).map(async (container: any) => { try { // 5 second timeout per container (increased from 2s for Hawser environments) const stats = await withTimeout( getContainerStats(container.id, env.id) as Promise, 5000, null ); if (!stats) return null; const cpuPercent = calculateCpuPercent(stats); const memoryUsage = stats.memory_stats?.usage || 0; const memoryLimit = stats.memory_stats?.limit || 1; const memoryPercent = (memoryUsage / memoryLimit) * 100; return { name: container.name, cpuPercent: Math.round(cpuPercent * 100) / 100, memoryPercent: Math.round(memoryPercent * 100) / 100 }; } catch { return null; } }); const topContainersResults = await Promise.all(topContainersPromises); envStats.topContainers = topContainersResults .filter((c): c is { name: string; cpuPercent: number; memoryPercent: number } => c !== null) .sort((a, b) => b.cpuPercent - a.cpuPercent) .slice(0, 10); envStats.loading!.topContainers = false; onPartialUpdate({ id: env.id, topContainers: [...envStats.topContainers], loading: { ...envStats.loading! } }); return envStats.topContainers; }); // Wait for all to complete await Promise.all([ containersPromise, imagesPromise, networksPromise, stacksPromise, diskUsagePromise, topContainersPromise ]); // Clear loading states when complete envStats.loading = undefined; } catch (error) { // Convert technical error messages to user-friendly ones const errorStr = String(error); if (errorStr.includes('not connected') || errorStr.includes('Edge agent')) { envStats.error = 'Agent not connected'; } else if (errorStr.includes('FailedToOpenSocket') || errorStr.includes('ECONNREFUSED')) { envStats.error = 'Docker socket not accessible'; } else if (errorStr.includes('ECONNRESET') || errorStr.includes('connection was closed')) { envStats.error = 'Connection lost'; } else if (errorStr.includes('verbose: true') || errorStr.includes('verbose')) { envStats.error = 'Connection failed'; } else if (errorStr.includes('timeout') || errorStr.includes('Timeout')) { envStats.error = 'Connection timeout'; } else { // Extract just the error message, not the full stack/details const match = errorStr.match(/^(?:Error:\s*)?([^.!?]+[.!?]?)/); envStats.error = match ? match[1].trim() : 'Connection error'; } envStats.loading = undefined; // Send offline status to client onPartialUpdate({ id: env.id, online: false, error: envStats.error, loading: undefined }); } return envStats; } export const GET: RequestHandler = async ({ cookies }) => { const auth = await authorize(cookies); if (auth.authEnabled && !await auth.can('environments', 'view')) { return new Response(JSON.stringify({ error: 'Permission denied' }), { status: 403, headers: { 'Content-Type': 'application/json' } }); } let environments = await getEnvironments(); // In enterprise mode, filter environments by user's accessible environments if (auth.authEnabled && auth.isEnterprise && auth.isAuthenticated && !auth.isAdmin) { const accessibleIds = await auth.getAccessibleEnvironmentIds(); // accessibleIds is null if user has access to all environments if (accessibleIds !== null) { environments = environments.filter(env => accessibleIds.includes(env.id)); } } // Create a readable stream that sends environment stats progressively let controllerClosed = false; const stream = new ReadableStream({ async start(controller) { const encoder = new TextEncoder(); // Safe enqueue that checks if controller is still open const safeEnqueue = (data: string) => { if (!controllerClosed) { try { controller.enqueue(encoder.encode(data)); } catch { controllerClosed = true; } } }; // First, send the list of environments so the UI can show skeletons with loading states const envList = environments.map(env => ({ id: env.id, name: env.name, host: env.host ?? undefined, port: env.port ?? undefined, icon: env.icon || 'globe', socketPath: env.socketPath ?? undefined, collectActivity: env.collectActivity, collectMetrics: env.collectMetrics ?? true, labels: parseLabels(env.labels), connectionType: (env.connectionType as 'socket' | 'direct' | 'hawser-standard' | 'hawser-edge') || 'socket', // Initial loading state for all sections loading: { containers: true, images: true, volumes: true, networks: true, stacks: true, diskUsage: true, topContainers: true } })); safeEnqueue(`event: environments\ndata: ${JSON.stringify(envList)}\n\n`); // Fetch stats for each environment with progressive updates const promises = environments.map(async (env) => { try { await getEnvironmentStatsProgressive(env, (partialStats) => { // Send partial update as it arrives safeEnqueue(`event: partial\ndata: ${JSON.stringify(partialStats)}\n\n`); }); // Send final complete stats event for this environment safeEnqueue(`event: complete\ndata: ${JSON.stringify({ id: env.id })}\n\n`); } catch (error) { console.error(`Failed to get stats for ${env.name}:`, error); // Convert technical error to user-friendly message const errorStr = String(error); let friendlyError = 'Connection error'; if (errorStr.includes('FailedToOpenSocket') || errorStr.includes('ECONNREFUSED')) { friendlyError = 'Docker socket not accessible'; } else if (errorStr.includes('ECONNRESET') || errorStr.includes('connection was closed')) { friendlyError = 'Connection lost'; } else if (errorStr.includes('verbose') || errorStr.includes('typo')) { friendlyError = 'Connection failed'; } else if (errorStr.includes('timeout') || errorStr.includes('Timeout')) { friendlyError = 'Connection timeout'; } safeEnqueue(`event: error\ndata: ${JSON.stringify({ id: env.id, error: friendlyError })}\n\n`); } }); // Wait for all to complete await Promise.all(promises); // Send done event and close if (!controllerClosed) { safeEnqueue(`event: done\ndata: {}\n\n`); try { controller.close(); } catch { // Already closed } } }, cancel() { // Called when the client disconnects controllerClosed = true; } }); return new Response(stream, { headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive' } }); };