Limit the content size of the blob index. (dart-lang#8610)

app/lib/task/backend.dart

@@ -5,6 +5,7 @@ import 'dart:typed_data';
 
 import 'package:_pub_shared/data/task_api.dart' as api;
 import 'package:_pub_shared/data/task_payload.dart';
+import 'package:_pub_shared/worker/limits.dart';
 import 'package:chunked_stream/chunked_stream.dart' show MaximumSizeExceeded;
 import 'package:clock/clock.dart';
 import 'package:collection/collection.dart';
@@ -788,7 +789,7 @@ class TaskBackend {
     try {
       return await _bucket.readAsBytes(
         path, offset: offset, length: length,
-        maxSize: 10 * 1024 * 1024, // sanity limit
+        maxSize: blobContentSizeLimit, // sanity limit
       );
     } on DetailedApiRequestError catch (e) {
       if (e.status == 404) {

pkg/_pub_shared/lib/worker/limits.dart

@@ -0,0 +1,6 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/// The maximum size of the compressed blob content that we want to store and serve.
+const blobContentSizeLimit = 10 * 1024 * 1024;

pkg/indexed_blob/lib/indexed_blob.dart

@@ -76,17 +76,36 @@ final class IndexedBlobBuilder {
   /// This cannot be called concurrently, callers must await this operation
   /// being completed.
   ///
+  /// When [skipAfterSize] is set, the blob file may contain the streamed content
+  /// up to the specified number of bytes, but will skip updating the index file
+  /// after the threshold is reached.
+  ///
   /// If an exception is thrown generated blob is not valid.
-  Future<void> addFile(String path, Stream<List<int>> content) async {
+  Future<void> addFile(
+    String path,
+    Stream<List<int>> content, {
+    int skipAfterSize = 0,
+  }) async {
     _checkState();
     try {
       _isAdding = true;
       final start = _offset;
+      var totalSize = 0;
       await _blob.addStream(content.map((chunk) {
+        totalSize += chunk.length;
+        if (skipAfterSize > 0 && totalSize > skipAfterSize) {
+          // Do not store the remaining chunks, we will not store the entry.
+          return const [];
+        }
+
         _offset += chunk.length;
         return chunk;
       }));
 
+      if (skipAfterSize > 0 && totalSize > skipAfterSize) {
+        return;
+      }
+
       var target = _index;
       final segments = path.split('/');
       for (var i = 0; i < segments.length - 1; i++) {

pkg/indexed_blob/test/blob_test.dart

@@ -2,6 +2,7 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
@@ -101,4 +102,44 @@ void main() {
     final files = index.files.toList();
     expect(files, hasLength(5));
   });
+
+  test('size limit', () async {
+    final controller = StreamController<List<int>>();
+    final result = controller.stream.toList();
+    final b = IndexedBlobBuilder(controller);
+    await b.addFile(
+      'a',
+      Stream.value([0, 1]),
+      skipAfterSize: 3,
+    );
+    await b.addFile(
+      'b',
+      Stream.fromIterable([
+        [0],
+        [1, 2, 3], // will be removed,
+      ]),
+      skipAfterSize: 3,
+    );
+    await b.addFile(
+      'c',
+      Stream.value([8, 9]),
+      skipAfterSize: 3,
+    );
+    final index = await b.buildIndex('1');
+    final files = index.files.toList();
+    expect(files, hasLength(2));
+    await controller.close();
+    expect(await result, [
+      [0, 1],
+      [0],
+      [],
+      [8, 9],
+    ]);
+
+    expect(index.lookup('b'), isNull);
+
+    final c = index.lookup('c')!;
+    expect(c.start, 3);
+    expect(c.end, 5);
+  });
 }

pkg/pub_worker/lib/src/analyze.dart

@@ -10,6 +10,7 @@ import 'dart:isolate' show Isolate;
 
 import 'package:_pub_shared/data/task_payload.dart';
 import 'package:_pub_shared/pubapi.dart';
+import 'package:_pub_shared/worker/limits.dart';
 import 'package:api_builder/api_builder.dart';
 import 'package:clock/clock.dart' show clock;
 import 'package:http/http.dart' show Client, ClientException;
@@ -201,7 +202,11 @@ Future<void> _analyzePackage(
           continue; // We'll add this at the very end!
         }
         try {
-          await builder.addFile(path, f.openRead().transform(gzip.encoder));
+          await builder.addFile(
+            path,
+            f.openRead().transform(gzip.encoder),
+            skipAfterSize: blobContentSizeLimit,
+          );
         } on IOException {
           log.writeln('ERROR: Failed to read output file at "$path"');
         }