diff --git a/packages/base-manager/src/utils.ts b/packages/base-manager/src/utils.ts index 0e0c7f59ba1..b8a26bad008 100644 --- a/packages/base-manager/src/utils.ts +++ b/packages/base-manager/src/utils.ts @@ -298,3 +298,16 @@ export function bufferToBase64(buffer: ArrayBuffer): string { export function base64ToBuffer(base64: string): ArrayBuffer { return toByteArray(base64).buffer; } + + +/** + * Map a function onto a list in batches, resolving each batch of returned + * promises before moving to the next batch. + */ +export async function mapBatch(list: T[], step: number, fn: (value: T, index: number, array: T[]) => Promise | U, thisArg?: any): Promise { + const results = []; + for(let i = 0; i < list.length; i+=step) { + results.push(...await Promise.all(list.slice(i, i+step).map(fn, thisArg))) + } + return results; +} diff --git a/packages/jupyterlab-manager/src/manager.ts b/packages/jupyterlab-manager/src/manager.ts index c1b306339a1..c7416b12702 100644 --- a/packages/jupyterlab-manager/src/manager.ts +++ b/packages/jupyterlab-manager/src/manager.ts @@ -19,7 +19,8 @@ import { import { ManagerBase, serialize_state, - IStateOptions + IStateOptions, + mapBatch } from '@jupyter-widgets/base-manager'; import { IDisposable } from '@lumino/disposable'; @@ -144,9 +145,11 @@ export abstract class LabWidgetManager extends ManagerBase } const comm_ids = await this._get_comm_info(); - // For each comm id that we do not know about, create the comm, and request the state. - const widgets_info = await Promise.all( - Object.keys(comm_ids).map(async comm_id => { + // For each comm id that we do not know about, create the comm, and + // request the state. We must do this in batches to make sure we do not + // exceed the ZMQ high water mark limiting messages from the kernel. See + // https://github.com/voila-dashboards/voila/issues/534 for more details. + const widgets_info = await mapBatch(Object.keys(comm_ids), 100, async comm_id => { try { await this.get_model(comm_id); // If we successfully get the model, do no more. @@ -192,8 +195,7 @@ export abstract class LabWidgetManager extends ManagerBase return info.promise; } - }) - ); + }); // We put in a synchronization barrier here so that we don't have to // topologically sort the restored widgets. `new_model` synchronously