From a8261009020fc87315d9542318451431d7e323c0 Mon Sep 17 00:00:00 2001 From: Matej Lednicky Date: Tue, 10 Oct 2023 11:02:34 +0200 Subject: [PATCH] feat(PR-40): auto-page list requests (#100) When calling `workspaces.list`, `forms.list`, `themes.list` or `responses.list` you can supply `page: "auto"` and the library will fetch all items across all pages for you automatically. It will fetch with maximum available `pageSize` to minimize number of requests. --- README.md | 4 ++ package.json | 1 + src/auto-page-items.ts | 47 ++++++++++++++++++++ src/bin.ts | 17 +++---- src/forms.ts | 30 ++++++++----- src/responses.ts | 83 ++++++++++++++++++++++++++++------- src/themes.ts | 26 +++++++---- src/utils.ts | 5 +++ src/workspaces.ts | 28 +++++++----- tests/unit/forms.test.ts | 30 ++++++++++++- tests/unit/responses.test.ts | 38 ++++++++++++++++ tests/unit/themes.test.ts | 26 +++++++++++ tests/unit/workspaces.test.ts | 32 ++++++++++++++ yarn.lock | 5 +++ 14 files changed, 316 insertions(+), 56 deletions(-) create mode 100644 src/auto-page-items.ts diff --git a/README.md b/README.md index 33f4bad..003937e 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,7 @@ Each one of them encapsulates the operations related to it (like listing, updati - Get a list of your typeforms - Returns a list of typeforms with the payload [referenced here](https://developer.typeform.com/create/reference/retrieve-forms/). +- You can set `page: "auto"` to automatically fetch all pages if there are more. It fetches with maximum `pageSize: 200`. #### `forms.get({ uid })` @@ -214,6 +215,7 @@ Each one of them encapsulates the operations related to it (like listing, updati - Gets your themes collection - `page`: default `1` + - set `page: "auto"` to automatically fetch all pages if there are more, it fetches with maximum `pageSize: 200` - `pageSize: default `10` #### `themes.get({ id })` @@ -262,6 +264,7 @@ Each one of them encapsulates the operations related to it (like listing, updati - Retrieve all workspaces in your account. - `page`: The page of results to retrieve. Default `1` is the first page of results. + - set `page: "auto"` to automatically fetch all pages if there are more, it fetches with maximum `pageSize: 200` - `pageSize`: Number of results to retrieve per page. Default is 10. Maximum is 200. - `search`: Returns items that contain the specified string. @@ -290,6 +293,7 @@ Each one of them encapsulates the operations related to it (like listing, updati - Returns form responses and date and time of form landing and submission. - `uid`: Unique ID for the form. - `pageSize`: Maximum number of responses. Default value is 25. Maximum value is 1000. +- `page`: Set to `"auto"` to automatically fetch all pages if there are more. It fetches with maximum `pageSize: 1000`. The `after` value is ignored when automatic paging is enabled. The responses will be sorted in the order that our system processed them (instead of the default order, `submitted_at`). **Note that it does not accept numeric value to identify page number.** - `since`: Limit request to responses submitted since the specified date and time. In ISO 8601 format, UTC time, to the second, with T as a delimiter between the date and time. - `until`: Limit request to responses submitted until the specified date and time. In ISO 8601 format, UTC time, to the second, with T as a delimiter between the date and time. - `after`: Limit request to responses submitted after the specified token. If you use the `after` parameter, the responses will be sorted in the order that our system processed them (instead of the default order, `submitted_at`). diff --git a/package.json b/package.json index 0dd3dbf..827114e 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,7 @@ "rollup-plugin-typescript2": "^0.24.1", "semantic-release": "^17.0.7", "ts-jest": "^24.0.2", + "tslib": "^2.6.2", "typescript": "^4.9.5" }, "jest": { diff --git a/src/auto-page-items.ts b/src/auto-page-items.ts new file mode 100644 index 0000000..008d8a2 --- /dev/null +++ b/src/auto-page-items.ts @@ -0,0 +1,47 @@ +import { rateLimit } from './utils' + +// request with maximum available page size to minimize number of requests +const MAX_PAGE_SIZE = 200 + +type RequestItemsFn = ( + page: number, + pageSize: number +) => Promise<{ + total_items: number + page_count: number + items: Item[] +}> + +const requestPageItems = async ( + requestFn: RequestItemsFn, + page = 1 +): Promise => { + await rateLimit() + const { items = [] } = (await requestFn(page, MAX_PAGE_SIZE)) || {} + const moreItems = + items.length === MAX_PAGE_SIZE + ? await requestPageItems(requestFn, page + 1) + : [] + return [...items, ...moreItems] +} + +export const autoPageItems = async ( + requestFn: RequestItemsFn +): Promise<{ + total_items: number + page_count: 1 + items: Item[] +}> => { + const { total_items = 0, items = [] } = + (await requestFn(1, MAX_PAGE_SIZE)) || {} + return { + total_items, + page_count: 1, + items: [ + ...items, + ...(total_items > items.length + ? await requestPageItems(requestFn, 2) + : []), + ], + } +} diff --git a/src/bin.ts b/src/bin.ts index e37bd23..94c1b1e 100644 --- a/src/bin.ts +++ b/src/bin.ts @@ -16,7 +16,7 @@ if (!token) { const typeformAPI = createClient({ token }) const [, , ...args] = process.argv -const [methodName, methodParams] = args +const [methodName, ...methodParams] = args if (!methodName || methodName === '-h' || methodName === '--help') { print('usage: typeform-api [params]') @@ -35,17 +35,18 @@ if (!typeformAPI[property]?.[method]) { let parsedParams = undefined -if (methodParams) { +if (methodParams && methodParams.length > 0) { + const methodParamsString = methodParams.join(',') + const normalizedParams = methodParamsString.startsWith('{') + ? methodParamsString + : `{${methodParamsString}}` + try { // this eval executes code supplied by user on their own machine, this is safe // eslint-disable-next-line no-eval - eval(`parsedParams = ${methodParams}`) + eval(`parsedParams = ${normalizedParams}`) } catch (err) { - throw new Error(`Invalid params: ${methodParams}`) - } - - if (typeof parsedParams !== 'object') { - throw new Error(`Invalid params: ${methodParams}`) + throw new Error(`Invalid params: ${normalizedParams}`) } } diff --git a/src/forms.ts b/src/forms.ts index 1716641..c8d483d 100644 --- a/src/forms.ts +++ b/src/forms.ts @@ -1,4 +1,5 @@ import { Typeform } from './typeform-types' +import { autoPageItems } from './auto-page-items' export class Forms { private _messages: FormMessages @@ -40,7 +41,7 @@ export class Forms { } public list(args?: { - page?: number + page?: number | 'auto' pageSize?: number search?: string workspaceId?: string @@ -52,16 +53,23 @@ export class Forms { workspaceId: null, } - return this._http.request({ - method: 'get', - url: `/forms`, - params: { - page, - page_size: pageSize, - search, - workspace_id: workspaceId, - }, - }) + const request = (page: number, pageSize: number) => + this._http.request({ + method: 'get', + url: `/forms`, + params: { + page, + page_size: pageSize, + search, + workspace_id: workspaceId, + }, + }) + + if (page === 'auto') { + return autoPageItems(request) + } + + return request(page, pageSize) } public update(args: { diff --git a/src/responses.ts b/src/responses.ts index 73198a1..c2fae86 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -1,4 +1,5 @@ import { Typeform } from './typeform-types' +import { rateLimit } from './utils' export class Responses { constructor(private _http: Typeform.HTTPClient) {} @@ -27,6 +28,7 @@ export class Responses { sort?: string query?: string fields?: string | string[] + page?: 'auto' }): Promise { const { uid, @@ -40,24 +42,32 @@ export class Responses { sort, query, fields, + page, } = args - return this._http.request({ - method: 'get', - url: `/forms/${uid}/responses`, - params: { - page_size: pageSize, - since, - until, - after, - before, - included_response_ids: toCSL(ids), - completed, - sort, - query, - fields: toCSL(fields), - }, - }) + const request = (pageSize: number, before: string) => + this._http.request({ + method: 'get', + url: `/forms/${uid}/responses`, + params: { + page_size: pageSize, + since, + until, + after, + before, + included_response_ids: toCSL(ids), + completed, + sort, + query, + fields: toCSL(fields), + }, + }) + + if (page === 'auto') { + return autoPageResponses(request) + } + + return request(pageSize, before) } } @@ -68,3 +78,44 @@ const toCSL = (args: string | string[]): string => { return typeof args === 'string' ? args : args.join(',') } + +// when auto-paginating, request with maximum available page size to minimize number of requests +const MAX_RESULTS_PAGE_SIZE = 1000 + +type RequestResultsFn = ( + pageSize: number, + before?: string +) => Promise + +const getLastResponseId = (items: Typeform.Response[]) => + items.length > 0 ? items[items.length - 1]?.response_id : null + +const requestPageResponses = async ( + requestFn: RequestResultsFn, + before: string = undefined +): Promise => { + await rateLimit() + const { items = [] } = (await requestFn(MAX_RESULTS_PAGE_SIZE, before)) || {} + const moreItems = + items.length === MAX_RESULTS_PAGE_SIZE + ? await requestPageResponses(requestFn, getLastResponseId(items)) + : [] + return [...items, ...moreItems] +} + +const autoPageResponses = async ( + requestFn: RequestResultsFn +): Promise => { + const { total_items = 0, items = [] } = + (await requestFn(MAX_RESULTS_PAGE_SIZE)) || {} + return { + total_items, + page_count: 1, + items: [ + ...items, + ...(total_items > items.length + ? await requestPageResponses(requestFn, getLastResponseId(items)) + : []), + ], + } +} diff --git a/src/themes.ts b/src/themes.ts index 5b24670..6110cff 100644 --- a/src/themes.ts +++ b/src/themes.ts @@ -1,5 +1,6 @@ import { Typeform } from './typeform-types' import { FONTS_AVAILABLE } from './constants' +import { autoPageItems } from './auto-page-items' export class Themes { constructor(private _http: Typeform.HTTPClient) {} @@ -34,19 +35,26 @@ export class Themes { } public list(args?: { - page?: number + page?: number | 'auto' pageSize?: number }): Promise { const { page, pageSize } = args || { page: null, pageSize: null } - return this._http.request({ - method: 'get', - url: '/themes', - params: { - page, - page_size: pageSize, - }, - }) + const request = (page: number, pageSize: number) => + this._http.request({ + method: 'get', + url: '/themes', + params: { + page, + page_size: pageSize, + }, + }) + + if (page === 'auto') { + return autoPageItems(request) + } + + return request(page, pageSize) } public update(args: { diff --git a/src/utils.ts b/src/utils.ts index 70d96d3..3fdc4db 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -16,3 +16,8 @@ export const createMemberPatchQuery = ( export const isMemberPropValid = (members: string | string[]): boolean => { return members && (typeof members === 'string' || Array.isArray(members)) } + +// two requests per second, per Typeform account +// https://www.typeform.com/developers/get-started/#rate-limits +export const rateLimit = () => + new Promise((resolve) => setTimeout(resolve, 500)) diff --git a/src/workspaces.ts b/src/workspaces.ts index 7699f02..08ad92f 100644 --- a/src/workspaces.ts +++ b/src/workspaces.ts @@ -1,5 +1,6 @@ import { Typeform } from './typeform-types' import { isMemberPropValid, createMemberPatchQuery } from './utils' +import { autoPageItems } from './auto-page-items' export class Workspaces { constructor(private _http: Typeform.HTTPClient) {} @@ -53,7 +54,7 @@ export class Workspaces { public list(args?: { search?: string - page?: number + page?: number | 'auto' pageSize?: number }): Promise { const { search, page, pageSize } = args || { @@ -62,15 +63,22 @@ export class Workspaces { pageSize: null, } - return this._http.request({ - method: 'get', - url: '/workspaces', - params: { - page, - page_size: pageSize, - search, - }, - }) + const request = (page: number, pageSize: number) => + this._http.request({ + method: 'get', + url: '/workspaces', + params: { + page, + page_size: pageSize, + search, + }, + }) + + if (page === 'auto') { + return autoPageItems(request) + } + + return request(page, pageSize) } public removeMembers(args: { diff --git a/tests/unit/forms.test.ts b/tests/unit/forms.test.ts index 4c898ec..8acfa80 100644 --- a/tests/unit/forms.test.ts +++ b/tests/unit/forms.test.ts @@ -5,7 +5,24 @@ import { Forms } from '../../src/forms' beforeEach(() => { axios.reset() - axios.onAny().reply(200) + + axios.onGet(`${API_BASE_URL}/forms?page=1&page_size=200`).replyOnce(200, { + total_items: 403, + page_count: 3, + items: Array.from({ length: 200 }, (_, i) => i), + }) + axios.onGet(`${API_BASE_URL}/forms?page=2&page_size=200`).replyOnce(200, { + total_items: 403, + page_count: 3, + items: Array.from({ length: 200 }, (_, i) => 200 + i), + }) + axios.onGet(`${API_BASE_URL}/forms?page=3&page_size=200`).replyOnce(200, { + total_items: 402, + page_count: 3, + items: [400, 401, 402], + }) + + axios.onAny().replyOnce(200, {}) }) const http = clientConstructor({ @@ -19,7 +36,7 @@ test('get all forms has the correct method and path', async () => { expect(axios.history.get[0].method).toBe('get') }) -test('paramters are sent correctly', async () => { +test('parameters are sent correctly', async () => { await formsRequest.list({ page: 2, pageSize: 10, @@ -34,6 +51,15 @@ test('paramters are sent correctly', async () => { expect(params.get('workspace_id')).toBe('abc') }) +test('get all forms with automatic pagination', async () => { + const list = await formsRequest.list({ page: 'auto' }) + expect(list).toEqual({ + total_items: 403, + page_count: 1, + items: Array.from({ length: 403 }, (_, i) => i), + }) +}) + test('getForm sends the correct UID', async () => { await formsRequest.get({ uid: 'abc123' }) expect(axios.history.get[0].url).toBe(`${API_BASE_URL}/forms/abc123`) diff --git a/tests/unit/responses.test.ts b/tests/unit/responses.test.ts index 8352b71..8b1bed5 100644 --- a/tests/unit/responses.test.ts +++ b/tests/unit/responses.test.ts @@ -5,6 +5,35 @@ import { Responses } from '../../src/responses' beforeEach(() => { axios.reset() + + axios + .onGet(`${API_BASE_URL}/forms/2/responses?page_size=1000`) + .replyOnce(200, { + total_items: 2003, + page_count: 3, + items: Array.from({ length: 1000 }, (_, i) => ({ response_id: i })), + }) + axios + .onGet(`${API_BASE_URL}/forms/2/responses?page_size=1000&before=999`) + .replyOnce(200, { + total_items: 2003, + page_count: 3, + items: Array.from({ length: 1000 }, (_, i) => ({ + response_id: 1000 + i, + })), + }) + axios + .onGet(`${API_BASE_URL}/forms/2/responses?page_size=1000&before=1999`) + .replyOnce(200, { + total_items: 2003, + page_count: 3, + items: [ + { response_id: 2000 }, + { response_id: 2001 }, + { response_id: 2002 }, + ], + }) + axios.onAny().reply(200) }) @@ -27,6 +56,15 @@ test('List responses with the given filters', async () => { expect(params.get('after')).toBe('12345') }) +test('List responses with automatic pagination', async () => { + const list = await responsesRequest.list({ uid: '2', page: 'auto' }) + expect(list).toEqual({ + total_items: 2003, + page_count: 1, + items: Array.from({ length: 2003 }, (_, i) => ({ response_id: i })), + }) +}) + test('Delete responses has the correct path and method when given string for `ids`', async () => { await responsesRequest.delete({ uid: '2', ids: '123' }) expect(axios.history.delete[0].url).toBe( diff --git a/tests/unit/themes.test.ts b/tests/unit/themes.test.ts index 5486d96..8d6ebd2 100644 --- a/tests/unit/themes.test.ts +++ b/tests/unit/themes.test.ts @@ -17,6 +17,23 @@ const mockThemePayload = { beforeEach(() => { axios.reset() + + axios.onGet(`${API_BASE_URL}/themes?page=1&page_size=200`).replyOnce(200, { + total_items: 403, + page_count: 3, + items: Array.from({ length: 200 }, (_, i) => i), + }) + axios.onGet(`${API_BASE_URL}/themes?page=2&page_size=200`).replyOnce(200, { + total_items: 403, + page_count: 3, + items: Array.from({ length: 200 }, (_, i) => 200 + i), + }) + axios.onGet(`${API_BASE_URL}/themes?page=3&page_size=200`).replyOnce(200, { + total_items: 402, + page_count: 3, + items: [400, 401, 402], + }) + axios.onAny().reply(200) }) @@ -37,6 +54,15 @@ test('Get themes has the correct parameters', async () => { expect(params.get('page_size')).toBe('15') }) +test('get all themes with automatic pagination', async () => { + const list = await themesRequest.list({ page: 'auto' }) + expect(list).toEqual({ + total_items: 403, + page_count: 1, + items: Array.from({ length: 403 }, (_, i) => i), + }) +}) + test('Get themes has the correct path', async () => { await themesRequest.get({ id: '2' }) expect(axios.history.get[0].url).toBe(`${API_BASE_URL}/themes/2`) diff --git a/tests/unit/workspaces.test.ts b/tests/unit/workspaces.test.ts index 6c0de2e..12e38a1 100644 --- a/tests/unit/workspaces.test.ts +++ b/tests/unit/workspaces.test.ts @@ -5,6 +5,29 @@ import { Workspaces } from '../../src/workspaces' beforeEach(() => { axios.reset() + + axios + .onGet(`${API_BASE_URL}/workspaces?page=1&page_size=200`) + .replyOnce(200, { + total_items: 403, + page_count: 3, + items: Array.from({ length: 200 }, (_, i) => i), + }) + axios + .onGet(`${API_BASE_URL}/workspaces?page=2&page_size=200`) + .replyOnce(200, { + total_items: 403, + page_count: 3, + items: Array.from({ length: 200 }, (_, i) => 200 + i), + }) + axios + .onGet(`${API_BASE_URL}/workspaces?page=3&page_size=200`) + .replyOnce(200, { + total_items: 402, + page_count: 3, + items: [400, 401, 402], + }) + axios.onAny().reply(200) }) @@ -30,6 +53,15 @@ test(`Get workspaces has the correct query parameters`, async () => { expect(params.get('page_size')).toBe('10') }) +test('get all workspaces with automatic pagination', async () => { + const list = await workspacesRequest.list({ page: 'auto' }) + expect(list).toEqual({ + total_items: 403, + page_count: 1, + items: Array.from({ length: 403 }, (_, i) => i), + }) +}) + test(`Get specific workscape has the correct path and method`, async () => { await workspacesRequest.get({ id: '2' }) expect(axios.history.get[0].url).toBe(`${API_BASE_URL}/workspaces/2`) diff --git a/yarn.lock b/yarn.lock index cc8c2ae..b388af4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11210,6 +11210,11 @@ tslib@1.10.0, tslib@^1.8.1: resolved "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz" integrity sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ== +tslib@^2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + tsutils@^3.21.0: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"