-
Notifications
You must be signed in to change notification settings - Fork 3.4k
fix: bound memory growth in loop executions to prevent OOM #3486
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,122 @@ | ||
| import { loggerMock } from '@sim/testing' | ||
| import { describe, expect, it, vi } from 'vitest' | ||
| import { DEFAULTS } from '@/executor/constants' | ||
| import type { LoopScope } from '@/executor/execution/state' | ||
| import type { ExecutionContext, NormalizedBlockOutput } from '@/executor/types' | ||
|
|
||
| vi.mock('@sim/logger', () => loggerMock) | ||
|
|
||
| /** | ||
| * Tests for memory bounds in loop execution (issue #2525). | ||
| * | ||
| * When loops run with many iterations (especially with agent blocks making | ||
| * tool calls), allIterationOutputs and blockLogs can grow unbounded, | ||
| * causing OOM on systems with limited memory. | ||
| */ | ||
|
|
||
| function createMinimalContext(overrides: Partial<ExecutionContext> = {}): ExecutionContext { | ||
| return { | ||
| workflowId: 'test-workflow', | ||
| blockStates: new Map(), | ||
| executedBlocks: new Set(), | ||
| blockLogs: [], | ||
| metadata: { duration: 0 }, | ||
| environmentVariables: {}, | ||
| decisions: { router: new Map(), condition: new Map() }, | ||
| completedLoops: new Set(), | ||
| activeExecutionPath: new Set(), | ||
| ...overrides, | ||
| } | ||
| } | ||
|
|
||
| describe('Loop memory bounds', () => { | ||
| describe('allIterationOutputs sliding window', () => { | ||
| it('should keep at most MAX_LOOP_ITERATION_HISTORY entries', () => { | ||
| const scope: LoopScope = { | ||
| iteration: 0, | ||
| currentIterationOutputs: new Map(), | ||
| allIterationOutputs: [], | ||
| } | ||
|
|
||
| const limit = DEFAULTS.MAX_LOOP_ITERATION_HISTORY | ||
|
|
||
| // Simulate more iterations than the limit | ||
| for (let i = 0; i < limit + 50; i++) { | ||
| const output: NormalizedBlockOutput = { content: `iteration-${i}` } | ||
| const iterationResults = [output] | ||
| scope.allIterationOutputs.push(iterationResults) | ||
|
|
||
| // Apply the same sliding window logic as loop.ts | ||
| if (scope.allIterationOutputs.length > limit) { | ||
| const excess = scope.allIterationOutputs.length - limit | ||
| scope.allIterationOutputs.splice(0, excess) | ||
| } | ||
| } | ||
|
|
||
| expect(scope.allIterationOutputs.length).toBe(limit) | ||
| // The oldest retained entry should be from iteration 50 | ||
| expect(scope.allIterationOutputs[0][0].content).toBe('iteration-50') | ||
| // The newest entry should be the last one pushed | ||
| expect(scope.allIterationOutputs[limit - 1][0].content).toBe( | ||
| `iteration-${limit + 49}` | ||
| ) | ||
| }) | ||
|
|
||
| it('should not prune when under the limit', () => { | ||
| const scope: LoopScope = { | ||
| iteration: 0, | ||
| currentIterationOutputs: new Map(), | ||
| allIterationOutputs: [], | ||
| } | ||
|
|
||
| for (let i = 0; i < 10; i++) { | ||
| scope.allIterationOutputs.push([{ content: `iter-${i}` }]) | ||
| } | ||
|
|
||
| expect(scope.allIterationOutputs.length).toBe(10) | ||
| expect(scope.allIterationOutputs[0][0].content).toBe('iter-0') | ||
| }) | ||
| }) | ||
|
|
||
| describe('blockLogs pruning', () => { | ||
| it('should keep at most MAX_BLOCK_LOGS entries', () => { | ||
| const ctx = createMinimalContext() | ||
| const limit = DEFAULTS.MAX_BLOCK_LOGS | ||
|
|
||
| // Simulate pushing more logs than the limit | ||
| for (let i = 0; i < limit + 100; i++) { | ||
| ctx.blockLogs.push({ | ||
| blockId: `block-${i}`, | ||
| blockType: 'function', | ||
| startedAt: new Date().toISOString(), | ||
| endedAt: new Date().toISOString(), | ||
| durationMs: 1, | ||
| success: true, | ||
| executionOrder: i + 1, | ||
| }) | ||
|
|
||
| // Apply the same pruning logic as block-executor.ts | ||
| if (ctx.blockLogs.length > limit) { | ||
| const excess = ctx.blockLogs.length - limit | ||
| ctx.blockLogs.splice(0, excess) | ||
| } | ||
| } | ||
|
|
||
| expect(ctx.blockLogs.length).toBe(limit) | ||
| // The oldest retained log should be from index 100 | ||
| expect(ctx.blockLogs[0].blockId).toBe('block-100') | ||
| }) | ||
| }) | ||
|
|
||
| describe('DEFAULTS constants', () => { | ||
| it('should define MAX_LOOP_ITERATION_HISTORY', () => { | ||
| expect(DEFAULTS.MAX_LOOP_ITERATION_HISTORY).toBeGreaterThan(0) | ||
| expect(typeof DEFAULTS.MAX_LOOP_ITERATION_HISTORY).toBe('number') | ||
| }) | ||
|
|
||
| it('should define MAX_BLOCK_LOGS', () => { | ||
| expect(DEFAULTS.MAX_BLOCK_LOGS).toBeGreaterThan(0) | ||
| expect(typeof DEFAULTS.MAX_BLOCK_LOGS).toBe('number') | ||
| }) | ||
| }) | ||
| }) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -248,6 +248,19 @@ export class LoopOrchestrator { | |
|
|
||
| if (iterationResults.length > 0) { | ||
| scope.allIterationOutputs.push(iterationResults) | ||
| scope.totalIterationCount = (scope.totalIterationCount ?? 0) + 1 | ||
|
|
||
| // Sliding window: discard oldest iteration outputs to bound memory (fixes #2525) | ||
| if (scope.allIterationOutputs.length > DEFAULTS.MAX_LOOP_ITERATION_HISTORY) { | ||
| const excess = scope.allIterationOutputs.length - DEFAULTS.MAX_LOOP_ITERATION_HISTORY | ||
| scope.allIterationOutputs.splice(0, excess) | ||
| logger.warn('Loop iteration history truncated to prevent OOM', { | ||
| loopId, | ||
| totalIterations: scope.totalIterationCount, | ||
| retained: DEFAULTS.MAX_LOOP_ITERATION_HISTORY, | ||
| discarded: scope.totalIterationCount - DEFAULTS.MAX_LOOP_ITERATION_HISTORY, | ||
| }) | ||
| } | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Iteration counter only tracks output-producing iterationsLow Severity
Additional Locations (1) |
||
| } | ||
|
|
||
| scope.currentIterationOutputs.clear() | ||
|
|
@@ -275,7 +288,12 @@ export class LoopOrchestrator { | |
| scope: LoopScope | ||
| ): LoopContinuationResult { | ||
| const results = scope.allIterationOutputs | ||
| const output = { results } | ||
| const totalIterations = scope.totalIterationCount ?? results.length | ||
| const output = { | ||
| results, | ||
| totalIterations, | ||
| ...(totalIterations > results.length && { truncated: true }), | ||
| } | ||
| this.state.setBlockOutput(loopId, output, DEFAULTS.EXECUTION_TIME) | ||
|
|
||
| if (this.contextExtensions?.onBlockComplete) { | ||
|
|
||


Uh oh!
There was an error while loading. Please reload this page.