From 1365a5f6fc4f26a250f6b7af373301ae2e2e9653 Mon Sep 17 00:00:00 2001 From: Maxwell Calkin Date: Mon, 9 Mar 2026 03:27:30 -0400 Subject: [PATCH 1/2] fix: bound memory growth in loop executions to prevent OOM (#2525) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When workflows with loops containing agent blocks run many iterations, three memory accumulation points cause unbounded growth leading to OOM: 1. allIterationOutputs grows without limit — every iteration's block outputs are retained forever. Fix: sliding window keeps only the last MAX_LOOP_ITERATION_HISTORY (100) iterations. 2. blockLogs accumulates every block execution with full input/output. With 1000 iterations × 5 blocks = 5000+ logs at 50KB+ each. Fix: prune oldest logs when exceeding MAX_BLOCK_LOGS (10,000). 3. Stream chunk buffers are never released after joining. Fix: clear the chunks array immediately after concatenation to allow GC. Co-Authored-By: Claude Opus 4.6 --- apps/sim/executor/constants.ts | 10 ++ apps/sim/executor/execution/block-executor.ts | 8 ++ .../orchestrators/loop-memory.test.ts | 122 ++++++++++++++++++ apps/sim/executor/orchestrators/loop.ts | 6 + 4 files changed, 146 insertions(+) create mode 100644 apps/sim/executor/orchestrators/loop-memory.test.ts diff --git a/apps/sim/executor/constants.ts b/apps/sim/executor/constants.ts index 65dafebd8dc..d02a1109438 100644 --- a/apps/sim/executor/constants.ts +++ b/apps/sim/executor/constants.ts @@ -159,6 +159,16 @@ export const DEFAULTS = { MAX_FOREACH_ITEMS: 1000, MAX_PARALLEL_BRANCHES: 20, MAX_NESTING_DEPTH: 10, + /** + * Maximum number of past iteration outputs retained in allIterationOutputs. + * Older entries are discarded (sliding window) to bound memory during long loops. + */ + MAX_LOOP_ITERATION_HISTORY: 100, + /** + * Maximum number of block logs retained during execution. + * When exceeded, the oldest logs are dropped to prevent OOM in long-running loops. + */ + MAX_BLOCK_LOGS: 10_000, /** Maximum child workflow depth for propagating SSE callbacks (block:started, block:completed). */ MAX_SSE_CHILD_DEPTH: 3, EXECUTION_TIME: 0, diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index 9c54d2bd993..a99e04bdfd8 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -77,6 +77,13 @@ export class BlockExecutor { if (!isSentinel) { blockLog = this.createBlockLog(ctx, node.id, block, node) ctx.blockLogs.push(blockLog) + + // Prune oldest block logs to bound memory in long-running loops (fixes #2525) + if (ctx.blockLogs.length > DEFAULTS.MAX_BLOCK_LOGS) { + const excess = ctx.blockLogs.length - DEFAULTS.MAX_BLOCK_LOGS + ctx.blockLogs.splice(0, excess) + } + this.callOnBlockStart(ctx, node, block, blockLog.executionOrder) } @@ -679,6 +686,7 @@ export class BlockExecutor { } const fullContent = chunks.join('') + chunks.length = 0 // Release chunk references to allow GC (fixes #2525) if (!fullContent) { return } diff --git a/apps/sim/executor/orchestrators/loop-memory.test.ts b/apps/sim/executor/orchestrators/loop-memory.test.ts new file mode 100644 index 00000000000..301695ab848 --- /dev/null +++ b/apps/sim/executor/orchestrators/loop-memory.test.ts @@ -0,0 +1,122 @@ +import { loggerMock } from '@sim/testing' +import { describe, expect, it, vi } from 'vitest' +import { DEFAULTS } from '@/executor/constants' +import type { LoopScope } from '@/executor/execution/state' +import type { ExecutionContext, NormalizedBlockOutput } from '@/executor/types' + +vi.mock('@sim/logger', () => loggerMock) + +/** + * Tests for memory bounds in loop execution (issue #2525). + * + * When loops run with many iterations (especially with agent blocks making + * tool calls), allIterationOutputs and blockLogs can grow unbounded, + * causing OOM on systems with limited memory. + */ + +function createMinimalContext(overrides: Partial = {}): ExecutionContext { + return { + workflowId: 'test-workflow', + blockStates: new Map(), + executedBlocks: new Set(), + blockLogs: [], + metadata: { duration: 0 }, + environmentVariables: {}, + decisions: { router: new Map(), condition: new Map() }, + completedLoops: new Set(), + activeExecutionPath: new Set(), + ...overrides, + } +} + +describe('Loop memory bounds', () => { + describe('allIterationOutputs sliding window', () => { + it('should keep at most MAX_LOOP_ITERATION_HISTORY entries', () => { + const scope: LoopScope = { + iteration: 0, + currentIterationOutputs: new Map(), + allIterationOutputs: [], + } + + const limit = DEFAULTS.MAX_LOOP_ITERATION_HISTORY + + // Simulate more iterations than the limit + for (let i = 0; i < limit + 50; i++) { + const output: NormalizedBlockOutput = { content: `iteration-${i}` } + const iterationResults = [output] + scope.allIterationOutputs.push(iterationResults) + + // Apply the same sliding window logic as loop.ts + if (scope.allIterationOutputs.length > limit) { + const excess = scope.allIterationOutputs.length - limit + scope.allIterationOutputs.splice(0, excess) + } + } + + expect(scope.allIterationOutputs.length).toBe(limit) + // The oldest retained entry should be from iteration 50 + expect(scope.allIterationOutputs[0][0].content).toBe('iteration-50') + // The newest entry should be the last one pushed + expect(scope.allIterationOutputs[limit - 1][0].content).toBe( + `iteration-${limit + 49}` + ) + }) + + it('should not prune when under the limit', () => { + const scope: LoopScope = { + iteration: 0, + currentIterationOutputs: new Map(), + allIterationOutputs: [], + } + + for (let i = 0; i < 10; i++) { + scope.allIterationOutputs.push([{ content: `iter-${i}` }]) + } + + expect(scope.allIterationOutputs.length).toBe(10) + expect(scope.allIterationOutputs[0][0].content).toBe('iter-0') + }) + }) + + describe('blockLogs pruning', () => { + it('should keep at most MAX_BLOCK_LOGS entries', () => { + const ctx = createMinimalContext() + const limit = DEFAULTS.MAX_BLOCK_LOGS + + // Simulate pushing more logs than the limit + for (let i = 0; i < limit + 100; i++) { + ctx.blockLogs.push({ + blockId: `block-${i}`, + blockType: 'function', + startedAt: new Date().toISOString(), + endedAt: new Date().toISOString(), + durationMs: 1, + success: true, + executionOrder: i + 1, + }) + + // Apply the same pruning logic as block-executor.ts + if (ctx.blockLogs.length > limit) { + const excess = ctx.blockLogs.length - limit + ctx.blockLogs.splice(0, excess) + } + } + + expect(ctx.blockLogs.length).toBe(limit) + // The oldest retained log should be from index 100 + expect(ctx.blockLogs[0].blockId).toBe('block-100') + }) + }) + + describe('DEFAULTS constants', () => { + it('should define MAX_LOOP_ITERATION_HISTORY', () => { + expect(DEFAULTS.MAX_LOOP_ITERATION_HISTORY).toBeGreaterThan(0) + expect(typeof DEFAULTS.MAX_LOOP_ITERATION_HISTORY).toBe('number') + }) + + it('should define MAX_BLOCK_LOGS', () => { + expect(DEFAULTS.MAX_BLOCK_LOGS).toBeGreaterThan(0) + expect(typeof DEFAULTS.MAX_BLOCK_LOGS).toBe('number') + }) + }) +}) diff --git a/apps/sim/executor/orchestrators/loop.ts b/apps/sim/executor/orchestrators/loop.ts index 039203c068a..d3ad15d34cf 100644 --- a/apps/sim/executor/orchestrators/loop.ts +++ b/apps/sim/executor/orchestrators/loop.ts @@ -248,6 +248,12 @@ export class LoopOrchestrator { if (iterationResults.length > 0) { scope.allIterationOutputs.push(iterationResults) + + // Sliding window: discard oldest iteration outputs to bound memory (fixes #2525) + if (scope.allIterationOutputs.length > DEFAULTS.MAX_LOOP_ITERATION_HISTORY) { + const excess = scope.allIterationOutputs.length - DEFAULTS.MAX_LOOP_ITERATION_HISTORY + scope.allIterationOutputs.splice(0, excess) + } } scope.currentIterationOutputs.clear() From f3b9f52e205677a44a03175e26b79c44c4d9e66a Mon Sep 17 00:00:00 2001 From: Maxwell Calkin Date: Mon, 9 Mar 2026 03:48:06 -0400 Subject: [PATCH 2/2] fix: add truncation awareness to loop output and warn on data loss Address review feedback: the sliding window on allIterationOutputs silently truncated the final loop output passed to downstream blocks. - Track totalIterationCount on LoopScope so the true iteration count is preserved even after pruning - Include totalIterations and truncated flag in the loop block output so downstream blocks can detect when data was pruned - Log a warning with discarded/retained counts when truncation occurs Co-Authored-By: Claude Opus 4.6 --- apps/sim/executor/execution/state.ts | 2 ++ apps/sim/executor/orchestrators/loop.ts | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/apps/sim/executor/execution/state.ts b/apps/sim/executor/execution/state.ts index bbbc7bc42c2..e5c8e05befa 100644 --- a/apps/sim/executor/execution/state.ts +++ b/apps/sim/executor/execution/state.ts @@ -16,6 +16,8 @@ export interface LoopScope { skipFirstConditionCheck?: boolean /** Error message if loop validation failed (e.g., exceeded max iterations) */ validationError?: string + /** Total number of iterations completed (may exceed allIterationOutputs.length when truncated) */ + totalIterationCount?: number } export interface ParallelScope { diff --git a/apps/sim/executor/orchestrators/loop.ts b/apps/sim/executor/orchestrators/loop.ts index d3ad15d34cf..fbc07385a4b 100644 --- a/apps/sim/executor/orchestrators/loop.ts +++ b/apps/sim/executor/orchestrators/loop.ts @@ -248,11 +248,18 @@ export class LoopOrchestrator { if (iterationResults.length > 0) { scope.allIterationOutputs.push(iterationResults) + scope.totalIterationCount = (scope.totalIterationCount ?? 0) + 1 // Sliding window: discard oldest iteration outputs to bound memory (fixes #2525) if (scope.allIterationOutputs.length > DEFAULTS.MAX_LOOP_ITERATION_HISTORY) { const excess = scope.allIterationOutputs.length - DEFAULTS.MAX_LOOP_ITERATION_HISTORY scope.allIterationOutputs.splice(0, excess) + logger.warn('Loop iteration history truncated to prevent OOM', { + loopId, + totalIterations: scope.totalIterationCount, + retained: DEFAULTS.MAX_LOOP_ITERATION_HISTORY, + discarded: scope.totalIterationCount - DEFAULTS.MAX_LOOP_ITERATION_HISTORY, + }) } } @@ -281,7 +288,12 @@ export class LoopOrchestrator { scope: LoopScope ): LoopContinuationResult { const results = scope.allIterationOutputs - const output = { results } + const totalIterations = scope.totalIterationCount ?? results.length + const output = { + results, + totalIterations, + ...(totalIterations > results.length && { truncated: true }), + } this.state.setBlockOutput(loopId, output, DEFAULTS.EXECUTION_TIME) if (this.contextExtensions?.onBlockComplete) {