Skip to content

Commit

Permalink
rate limit stream errors
Browse files Browse the repository at this point in the history
  • Loading branch information
@glbrntt
glbrntt committed Jan 6, 2025
1 parent 2c05ee8 commit 7527392
Show file tree
Hide file tree
Showing 5 changed files with 249 additions and 35 deletions.
17 changes: 17 additions & 0 deletions Sources/NIOHTTP2/DOSHeuristics.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,14 @@ struct DOSHeuristics<DeadlineClock: NIODeadlineClock> {
private let maximumSequentialEmptyDataFrames: Int

private var resetFrameRateControlStateMachine: RateLimitStateMachine
private var streamErrorRateControlStateMachine: RateLimitStateMachine

internal init(
maximumSequentialEmptyDataFrames: Int,
maximumResetFrameCount: Int,
resetFrameCounterWindow: TimeAmount,
maximumStreamErrorCount: Int,
streamErrorCounterWindow: TimeAmount,
clock: DeadlineClock = RealNIODeadlineClock()
) {
precondition(
Expand All @@ -47,6 +50,11 @@ struct DOSHeuristics<DeadlineClock: NIODeadlineClock> {
timeWindow: resetFrameCounterWindow,
clock: clock
)
self.streamErrorRateControlStateMachine = .init(
countThreshold: maximumStreamErrorCount,
timeWindow: streamErrorCounterWindow,
clock: clock
)
}
}

Expand Down Expand Up @@ -80,6 +88,15 @@ extension DOSHeuristics {
throw NIOHTTP2Errors.excessiveEmptyDataFrames()
}
}

mutating func processStreamError() throws {
switch self.streamErrorRateControlStateMachine.recordEvent() {
case .rateTooHigh:
throw NIOHTTP2Errors.excessiveStreamErrors()
case .noneReceived, .ratePermitted:
()
}
}
}

extension DOSHeuristics {
Expand Down
52 changes: 43 additions & 9 deletions Sources/NIOHTTP2/HTTP2ChannelHandler.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,9 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
maximumBufferedControlFrames: 10000,
maximumSequentialContinuationFrames: NIOHTTP2Handler.defaultMaximumSequentialContinuationFrames,
maximumResetFrameCount: 200,
resetFrameCounterWindow: .seconds(30)
resetFrameCounterWindow: .seconds(30),
maximumStreamErrorCount: 200,
streamErrorCounterWindow: .seconds(30)
)
}

Expand Down Expand Up @@ -268,7 +270,9 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
maximumBufferedControlFrames: maximumBufferedControlFrames,
maximumSequentialContinuationFrames: NIOHTTP2Handler.defaultMaximumSequentialContinuationFrames,
maximumResetFrameCount: 200,
resetFrameCounterWindow: .seconds(30)
resetFrameCounterWindow: .seconds(30),
maximumStreamErrorCount: 200,
streamErrorCounterWindow: .seconds(30)
)

}
Expand All @@ -294,7 +298,9 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
maximumBufferedControlFrames: connectionConfiguration.maximumBufferedControlFrames,
maximumSequentialContinuationFrames: connectionConfiguration.maximumSequentialContinuationFrames,
maximumResetFrameCount: streamConfiguration.streamResetFrameRateLimit.maximumCount,
resetFrameCounterWindow: streamConfiguration.streamResetFrameRateLimit.windowLength
resetFrameCounterWindow: streamConfiguration.streamResetFrameRateLimit.windowLength,
maximumStreamErrorCount: streamConfiguration.streamErrorRateLimit.maximumCount,
streamErrorCounterWindow: streamConfiguration.streamErrorRateLimit.windowLength
)
}

Expand All @@ -308,7 +314,9 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
maximumBufferedControlFrames: Int,
maximumSequentialContinuationFrames: Int,
maximumResetFrameCount: Int,
resetFrameCounterWindow: TimeAmount
resetFrameCounterWindow: TimeAmount,
maximumStreamErrorCount: Int,
streamErrorCounterWindow: TimeAmount
) {
self._eventLoop = eventLoop
self.stateMachine = HTTP2ConnectionStateMachine(
Expand All @@ -326,7 +334,9 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
self.denialOfServiceValidator = DOSHeuristics(
maximumSequentialEmptyDataFrames: maximumSequentialEmptyDataFrames,
maximumResetFrameCount: maximumResetFrameCount,
resetFrameCounterWindow: resetFrameCounterWindow
resetFrameCounterWindow: resetFrameCounterWindow,
maximumStreamErrorCount: maximumStreamErrorCount,
streamErrorCounterWindow: streamErrorCounterWindow
)
self.tolerateImpossibleStateTransitionsInDebugMode = false
self.inboundStreamMultiplexerState = .uninitializedLegacy
Expand Down Expand Up @@ -362,7 +372,9 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
maximumSequentialContinuationFrames: Int = NIOHTTP2Handler.defaultMaximumSequentialContinuationFrames,
tolerateImpossibleStateTransitionsInDebugMode: Bool = false,
maximumResetFrameCount: Int = 200,
resetFrameCounterWindow: TimeAmount = .seconds(30)
resetFrameCounterWindow: TimeAmount = .seconds(30),
maximumStreamErrorCount: Int = 200,
streamErrorCounterWindow: TimeAmount = .seconds(30)
) {
self.stateMachine = HTTP2ConnectionStateMachine(
role: .init(mode),
Expand All @@ -380,7 +392,9 @@ public final class NIOHTTP2Handler: ChannelDuplexHandler {
self.denialOfServiceValidator = DOSHeuristics(
maximumSequentialEmptyDataFrames: maximumSequentialEmptyDataFrames,
maximumResetFrameCount: maximumResetFrameCount,
resetFrameCounterWindow: resetFrameCounterWindow
resetFrameCounterWindow: resetFrameCounterWindow,
maximumStreamErrorCount: maximumStreamErrorCount,
streamErrorCounterWindow: streamErrorCounterWindow
)
self.tolerateImpossibleStateTransitionsInDebugMode = tolerateImpossibleStateTransitionsInDebugMode
self.inboundStreamMultiplexerState = .uninitializedLegacy
Expand Down Expand Up @@ -788,6 +802,12 @@ extension NIOHTTP2Handler {
private func processDoSRisk(_ frame: HTTP2Frame, result: inout StateMachineResultWithEffect) {
do {
try self.denialOfServiceValidator.process(frame)
switch result.result {
case .streamError:
try self.denialOfServiceValidator.processStreamError()
case .succeed, .ignoreFrame, .connectionError:
()
}
} catch {
result.result = StateMachineResult.connectionError(underlyingError: error, type: .enhanceYourCalm)
result.effect = nil
Expand Down Expand Up @@ -1301,7 +1321,9 @@ extension NIOHTTP2Handler {
maximumBufferedControlFrames: connectionConfiguration.maximumBufferedControlFrames,
maximumSequentialContinuationFrames: connectionConfiguration.maximumSequentialContinuationFrames,
maximumResetFrameCount: streamConfiguration.streamResetFrameRateLimit.maximumCount,
resetFrameCounterWindow: streamConfiguration.streamResetFrameRateLimit.windowLength
resetFrameCounterWindow: streamConfiguration.streamResetFrameRateLimit.windowLength,
maximumStreamErrorCount: streamConfiguration.streamErrorRateLimit.maximumCount,
streamErrorCounterWindow: streamConfiguration.streamErrorRateLimit.windowLength
)

self.inboundStreamMultiplexerState = .uninitializedInline(
Expand Down Expand Up @@ -1330,7 +1352,9 @@ extension NIOHTTP2Handler {
maximumBufferedControlFrames: connectionConfiguration.maximumBufferedControlFrames,
maximumSequentialContinuationFrames: connectionConfiguration.maximumSequentialContinuationFrames,
maximumResetFrameCount: streamConfiguration.streamResetFrameRateLimit.maximumCount,
resetFrameCounterWindow: streamConfiguration.streamResetFrameRateLimit.windowLength
resetFrameCounterWindow: streamConfiguration.streamResetFrameRateLimit.windowLength,
maximumStreamErrorCount: streamConfiguration.streamErrorRateLimit.maximumCount,
streamErrorCounterWindow: streamConfiguration.streamErrorRateLimit.windowLength
)
self.inboundStreamMultiplexerState = .uninitializedAsync(
streamConfiguration,
Expand Down Expand Up @@ -1361,6 +1385,7 @@ extension NIOHTTP2Handler {
public var outboundBufferSizeHighWatermark: Int = 8196
public var outboundBufferSizeLowWatermark: Int = 4092
public var streamResetFrameRateLimit: StreamResetFrameRateLimitConfiguration = .init()
public var streamErrorRateLimit: StreamErrorRateLimitConfiguration = .init()
public init() {}
}

Expand All @@ -1374,6 +1399,15 @@ extension NIOHTTP2Handler {
public init() {}
}

/// Stream error rate limit configuration.
///
/// The settings that control the maximum permitted stream errors within a given time window.
public struct StreamErrorRateLimitConfiguration: Hashable, Sendable {
public var maximumCount: Int = 200
public var windowLength: TimeAmount = .seconds(30)
public init() {}
}

/// Overall connection and stream-level configuration.
public struct Configuration: Hashable, Sendable {
/// The settings that will be used when establishing the connection. These will be sent to the peer as part of the
Expand Down
32 changes: 32 additions & 0 deletions Sources/NIOHTTP2/HTTP2Error.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,18 @@ public enum NIOHTTP2Errors {
ExcessiveContinuationFrames(file: file, line: line)
}

/// Creates an ``ExcessiveStreamErrors`` error with appropriate source context.
///
/// - Parameters:
/// - file: Source file of the caller.
/// - line: Source line number of the caller.
public static func excessiveStreamErrors(
file: String = #fileID,
line: UInt = #line
) -> ExcessiveStreamErrors {
ExcessiveStreamErrors(file: file, line: line)
}

/// Creates a ``StreamError`` error with appropriate source context.
///
/// - Parameters:
Expand Down Expand Up @@ -2013,6 +2025,26 @@ public enum NIOHTTP2Errors {
true
}
}

/// A remote peer has sent too many frames which result in a stream error.
public struct ExcessiveStreamErrors: NIOHTTP2Error {
private let file: String
private let line: UInt

/// The location where the error was thrown.
public var location: String {
_location(file: self.file, line: self.line)
}

fileprivate init(file: String, line: UInt) {
self.file = file
self.line = line
}

public static func == (lhs: Self, rhs: Self) -> Bool {
true
}
}
}

/// This enum covers errors that are thrown internally for messaging reasons. These should
Expand Down
110 changes: 85 additions & 25 deletions Tests/NIOHTTP2Tests/DOSHeuristicsTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,30 @@ import XCTest
@testable import NIOHTTP2

final class DOSHeuristicsTests: XCTestCase {
func testRSTFramePermittedRate() throws {
private func makeDOSHeuristics(
maximumSequentialEmptyDataFrames: Int = 100,
maximumResetFrameCount: Int = 200,
resetFrameCounterWindow: TimeAmount = .seconds(30),
maximumStreamErrorCount: Int = 200,
streamErrorCounterWindow: TimeAmount = .seconds(30)
) -> (DOSHeuristics<TestClock>, TestClock) {
let testClock = TestClock()
var dosHeuristics = DOSHeuristics(
maximumSequentialEmptyDataFrames: 100,
maximumResetFrameCount: 200,
resetFrameCounterWindow: .seconds(30),
let dosHeuristics = DOSHeuristics(
maximumSequentialEmptyDataFrames: maximumSequentialEmptyDataFrames,
maximumResetFrameCount: maximumResetFrameCount,
resetFrameCounterWindow: resetFrameCounterWindow,
maximumStreamErrorCount: maximumStreamErrorCount,
streamErrorCounterWindow: streamErrorCounterWindow,
clock: testClock
)
return (dosHeuristics, testClock)
}

func testRSTFramePermittedRate() throws {
var (dosHeuristics, testClock) = self.makeDOSHeuristics(
maximumResetFrameCount: 200,
resetFrameCounterWindow: .seconds(30)
)

// more resets than allowed, but slow enough to be okay
for i in 0..<300 {
Expand All @@ -35,12 +51,9 @@ final class DOSHeuristicsTests: XCTestCase {
}

func testRSTFrameExcessiveRate() throws {
let testClock = TestClock()
var dosHeuristics = DOSHeuristics(
maximumSequentialEmptyDataFrames: 100,
var (dosHeuristics, testClock) = self.makeDOSHeuristics(
maximumResetFrameCount: 200,
resetFrameCounterWindow: .seconds(30),
clock: testClock
resetFrameCounterWindow: .seconds(30)
)

// up to the limit
Expand All @@ -56,12 +69,9 @@ final class DOSHeuristicsTests: XCTestCase {
}

func testRSTFrameGarbageCollects() throws {
let testClock = TestClock()
var dosHeuristics = DOSHeuristics(
maximumSequentialEmptyDataFrames: 100,
var (dosHeuristics, testClock) = self.makeDOSHeuristics(
maximumResetFrameCount: 200,
resetFrameCounterWindow: .seconds(30),
clock: testClock
resetFrameCounterWindow: .seconds(30)
)

// up to the limit
Expand All @@ -86,12 +96,9 @@ final class DOSHeuristicsTests: XCTestCase {
}

func testRSTFrameExcessiveRateConfigurableCount() throws {
let testClock = TestClock()
var dosHeuristics = DOSHeuristics(
maximumSequentialEmptyDataFrames: 100,
var (dosHeuristics, testClock) = self.makeDOSHeuristics(
maximumResetFrameCount: 400,
resetFrameCounterWindow: .seconds(30),
clock: testClock
resetFrameCounterWindow: .seconds(30)
)

// up to the limit
Expand All @@ -107,12 +114,9 @@ final class DOSHeuristicsTests: XCTestCase {
}

func testRSTFrameExcessiveRateConfigurableWindow() throws {
let testClock = TestClock()
var dosHeuristics = DOSHeuristics(
maximumSequentialEmptyDataFrames: 100,
var (dosHeuristics, testClock) = self.makeDOSHeuristics(
maximumResetFrameCount: 200,
resetFrameCounterWindow: .seconds(3600),
clock: testClock
resetFrameCounterWindow: .seconds(3600)
)

// up to the limit, previously slow enough to be okay but not with this window
Expand All @@ -126,6 +130,62 @@ final class DOSHeuristicsTests: XCTestCase {
try dosHeuristics.process(.init(streamID: HTTP2StreamID(201), payload: .rstStream(.cancel)))
)
}

func testStreamErrorPermittedRate() throws {
var (dosHeuristics, testClock) = self.makeDOSHeuristics(
maximumStreamErrorCount: 200,
streamErrorCounterWindow: .seconds(30)
)

// More stream errors than allowed, but slow enough to be okay
for _ in 0..<300 {
try dosHeuristics.processStreamError()
testClock.advance(by: .seconds(1))
}
}

func testStreamErrorExcessiveRate() throws {
var (dosHeuristics, testClock) = self.makeDOSHeuristics(
maximumStreamErrorCount: 200,
streamErrorCounterWindow: .seconds(30)
)

// Up to the limit
for _ in 0..<200 {
try dosHeuristics.processStreamError()
testClock.advance(by: .milliseconds(1))
}

// Over the limit
XCTAssertThrowsError(
try dosHeuristics.processStreamError()
)
}

func testStreamErrorGarbageCollects() throws {
var (dosHeuristics, testClock) = self.makeDOSHeuristics(
maximumStreamErrorCount: 200,
streamErrorCounterWindow: .seconds(30)
)

// Up to the limit
for _ in 0..<200 {
try dosHeuristics.processStreamError()
testClock.advance(by: .milliseconds(1))
}

// Clear out counter
testClock.advance(by: .seconds(30))

// Up to the limit
for _ in 0..<200 {
try dosHeuristics.processStreamError()
testClock.advance(by: .milliseconds(1))
}

// Over the limit
XCTAssertThrowsError(try dosHeuristics.processStreamError())
}
}

class TestClock: NIODeadlineClock {
Expand Down
Loading

0 comments on commit 7527392

Please sign in to comment.