From e372213992cefd39629a53625f22096585213a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20So=C3=B3s?= Date: Wed, 5 Mar 2025 13:28:44 +0100 Subject: [PATCH] Limit the content size of the blob index. (#8610) --- app/lib/task/backend.dart | 3 +- pkg/_pub_shared/lib/worker/limits.dart | 6 ++++ pkg/indexed_blob/lib/indexed_blob.dart | 21 ++++++++++++- pkg/indexed_blob/test/blob_test.dart | 41 ++++++++++++++++++++++++++ pkg/pub_worker/lib/src/analyze.dart | 7 ++++- 5 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 pkg/_pub_shared/lib/worker/limits.dart diff --git a/app/lib/task/backend.dart b/app/lib/task/backend.dart index 00fe04bab0..f54f54d074 100644 --- a/app/lib/task/backend.dart +++ b/app/lib/task/backend.dart @@ -5,6 +5,7 @@ import 'dart:typed_data'; import 'package:_pub_shared/data/task_api.dart' as api; import 'package:_pub_shared/data/task_payload.dart'; +import 'package:_pub_shared/worker/limits.dart'; import 'package:chunked_stream/chunked_stream.dart' show MaximumSizeExceeded; import 'package:clock/clock.dart'; import 'package:collection/collection.dart'; @@ -788,7 +789,7 @@ class TaskBackend { try { return await _bucket.readAsBytes( path, offset: offset, length: length, - maxSize: 10 * 1024 * 1024, // sanity limit + maxSize: blobContentSizeLimit, // sanity limit ); } on DetailedApiRequestError catch (e) { if (e.status == 404) { diff --git a/pkg/_pub_shared/lib/worker/limits.dart b/pkg/_pub_shared/lib/worker/limits.dart new file mode 100644 index 0000000000..02aed9ca34 --- /dev/null +++ b/pkg/_pub_shared/lib/worker/limits.dart @@ -0,0 +1,6 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// The maximum size of the compressed blob content that we want to store and serve. +const blobContentSizeLimit = 10 * 1024 * 1024; diff --git a/pkg/indexed_blob/lib/indexed_blob.dart b/pkg/indexed_blob/lib/indexed_blob.dart index 7e35109836..24cb1bab35 100644 --- a/pkg/indexed_blob/lib/indexed_blob.dart +++ b/pkg/indexed_blob/lib/indexed_blob.dart @@ -76,17 +76,36 @@ final class IndexedBlobBuilder { /// This cannot be called concurrently, callers must await this operation /// being completed. /// + /// When [skipAfterSize] is set, the blob file may contain the streamed content + /// up to the specified number of bytes, but will skip updating the index file + /// after the threshold is reached. + /// /// If an exception is thrown generated blob is not valid. - Future addFile(String path, Stream> content) async { + Future addFile( + String path, + Stream> content, { + int skipAfterSize = 0, + }) async { _checkState(); try { _isAdding = true; final start = _offset; + var totalSize = 0; await _blob.addStream(content.map((chunk) { + totalSize += chunk.length; + if (skipAfterSize > 0 && totalSize > skipAfterSize) { + // Do not store the remaining chunks, we will not store the entry. + return const []; + } + _offset += chunk.length; return chunk; })); + if (skipAfterSize > 0 && totalSize > skipAfterSize) { + return; + } + var target = _index; final segments = path.split('/'); for (var i = 0; i < segments.length - 1; i++) { diff --git a/pkg/indexed_blob/test/blob_test.dart b/pkg/indexed_blob/test/blob_test.dart index 4e3b252c5d..3f7c9bb7a5 100644 --- a/pkg/indexed_blob/test/blob_test.dart +++ b/pkg/indexed_blob/test/blob_test.dart @@ -2,6 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:async'; import 'dart:convert'; import 'dart:io'; @@ -101,4 +102,44 @@ void main() { final files = index.files.toList(); expect(files, hasLength(5)); }); + + test('size limit', () async { + final controller = StreamController>(); + final result = controller.stream.toList(); + final b = IndexedBlobBuilder(controller); + await b.addFile( + 'a', + Stream.value([0, 1]), + skipAfterSize: 3, + ); + await b.addFile( + 'b', + Stream.fromIterable([ + [0], + [1, 2, 3], // will be removed, + ]), + skipAfterSize: 3, + ); + await b.addFile( + 'c', + Stream.value([8, 9]), + skipAfterSize: 3, + ); + final index = await b.buildIndex('1'); + final files = index.files.toList(); + expect(files, hasLength(2)); + await controller.close(); + expect(await result, [ + [0, 1], + [0], + [], + [8, 9], + ]); + + expect(index.lookup('b'), isNull); + + final c = index.lookup('c')!; + expect(c.start, 3); + expect(c.end, 5); + }); } diff --git a/pkg/pub_worker/lib/src/analyze.dart b/pkg/pub_worker/lib/src/analyze.dart index 609ad83f61..ac20b82814 100644 --- a/pkg/pub_worker/lib/src/analyze.dart +++ b/pkg/pub_worker/lib/src/analyze.dart @@ -10,6 +10,7 @@ import 'dart:isolate' show Isolate; import 'package:_pub_shared/data/task_payload.dart'; import 'package:_pub_shared/pubapi.dart'; +import 'package:_pub_shared/worker/limits.dart'; import 'package:api_builder/api_builder.dart'; import 'package:clock/clock.dart' show clock; import 'package:http/http.dart' show Client, ClientException; @@ -201,7 +202,11 @@ Future _analyzePackage( continue; // We'll add this at the very end! } try { - await builder.addFile(path, f.openRead().transform(gzip.encoder)); + await builder.addFile( + path, + f.openRead().transform(gzip.encoder), + skipAfterSize: blobContentSizeLimit, + ); } on IOException { log.writeln('ERROR: Failed to read output file at "$path"'); }