Skip to content

Commit

Permalink
feat(suite): distribution for ab testing
Browse files Browse the repository at this point in the history
  • Loading branch information
adderpositive authored and tomasklim committed Jan 7, 2025
1 parent d6d3f3d commit fc56ea0
Show file tree
Hide file tree
Showing 8 changed files with 244 additions and 2 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import { ReactElement } from 'react';

import { ExperimentNameType } from '@suite-common/message-system';

import { useExperiment } from 'src/hooks/experiment/useExperiment';

interface ExperimentWrapperProps {
id: ExperimentNameType;
components: Array<{
variant: string;
element: ReactElement;
}>;
}

/**
* @param components first item in components is default
*/
export const ExperimentWrapper = ({
id,
components,
}: ExperimentWrapperProps): ReactElement | null => {
const { experiment, activeExperimentVariant } = useExperiment(id);
const areComponentsEmpty = !components.length;

if (areComponentsEmpty) return null;

const defaultComponent = components[0];
const experimentOrVariantNotFound = !experiment || !activeExperimentVariant;
const experimentAndComponentsMismatch = experiment?.groups.length !== components.length;

if (experimentOrVariantNotFound || experimentAndComponentsMismatch) {
return defaultComponent.element;
}

const activeComponent = components.find(
component => component.variant === activeExperimentVariant.variant,
);

return activeComponent?.element ?? defaultComponent.element;
};
26 changes: 26 additions & 0 deletions packages/suite/src/hooks/experiment/useExperiment.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { useMemo } from 'react';

import { selectAnalyticsInstanceId } from '@suite-common/analytics';
import {
ExperimentNameType,
experiments,
selectActiveExperimentGroup,
selectExperimentById,
} from '@suite-common/message-system';

import { useSelector } from 'src/hooks/suite';

export const useExperiment = (id: ExperimentNameType) => {
const experimentUuid = experiments[id];
const instanceId = useSelector(selectAnalyticsInstanceId);
const experiment = useSelector(selectExperimentById(experimentUuid));
const activeExperimentVariant = useMemo(
() => selectActiveExperimentGroup({ instanceId, experiment }),
[instanceId, experiment],
);

return {
experiment,
activeExperimentVariant,
};
};
21 changes: 21 additions & 0 deletions suite-common/message-system/src/experiment/__fixtures__/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import { getWeakRandomId } from '@trezor/utils';

import { ExperimentIdType } from '../experiments';

// getWeakRandomId is also used for generating instanceId
export const getArrayOfInstanceIds = (count: number) =>
Array.from({ length: count }, () => getWeakRandomId(10));

export const experimentTest = {
id: 'e2e8d05f-1469-4e47-9ab0-53544e5cad07' as ExperimentIdType,
groups: [
{
variant: 'A',
percentage: 20,
},
{
variant: 'B',
percentage: 80,
},
],
};
73 changes: 73 additions & 0 deletions suite-common/message-system/src/experiment/__tests__/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import {
getExperimentGroupByInclusion,
getInclusionFromInstanceId,
selectActiveExperimentGroup,
} from '../';
import { experimentTest, getArrayOfInstanceIds } from '../__fixtures__';
import { ExperimentIdType } from '../experiments';

describe('testing experiment utils', () => {
const experimentId = 'e2e8d05f-1469-4e47-9ab0-53544e5cad07' as ExperimentIdType;

it('test getInclusionFromInstanceId whether returns percentage between 0 and 99', () => {
const arrayOfIds = getArrayOfInstanceIds(100);
const isExistNumberOutOfRange = arrayOfIds.some(id => {
const percentage = getInclusionFromInstanceId(id, experimentId);

return percentage < 0 || percentage > 99;
});

expect(isExistNumberOutOfRange).toEqual(false);
});

it('test getExperimentGroupByInclusion whether instanceId is not in range of variants', () => {
const arrayOfIds = getArrayOfInstanceIds(100);
const isExistInstanceIdNotInVariantRange = arrayOfIds.some(id => {
const inclusion = getInclusionFromInstanceId(id, experimentId);
const group = getExperimentGroupByInclusion({
groups: experimentTest.groups,
inclusion,
});

return group === undefined;
});

expect(isExistInstanceIdNotInVariantRange).toEqual(false);
});

it('test selectActiveExperimentGroup share of variant inclusion', () => {
const deviation = 0.05;
const sampleSize = 1000;
let groupACount = 0;
let groupBCount = 0;

const arrayOfIds = getArrayOfInstanceIds(sampleSize);

arrayOfIds.forEach(id => {
const selectedGroup = selectActiveExperimentGroup({
experiment: experimentTest,
instanceId: id,
});

if (selectedGroup?.variant === 'A') {
groupACount += 1;
}

if (selectedGroup?.variant === 'B') {
groupBCount += 1;
}
});

const shareA = groupACount / sampleSize;
const shareB = groupBCount / sampleSize;

expect(shareA).toBeGreaterThanOrEqual(
experimentTest.groups[0].percentage / 100 - deviation,
);
expect(shareA).toBeLessThanOrEqual(experimentTest.groups[0].percentage / 100 + deviation);
expect(shareB).toBeGreaterThanOrEqual(
experimentTest.groups[1].percentage / 100 - deviation,
);
expect(shareB).toBeLessThanOrEqual(experimentTest.groups[1].percentage / 100 + deviation);
});
});
6 changes: 6 additions & 0 deletions suite-common/message-system/src/experiment/experiments.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
export const experiments = {
// e.g. orangeSendButton: 'fb0eb1bc-8ec3-44d4-98eb-53301d73d981',
} as const;

export type ExperimentNameType = keyof typeof experiments;
export type ExperimentIdType = (typeof experiments)[ExperimentNameType];
69 changes: 69 additions & 0 deletions suite-common/message-system/src/experiment/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import { createHash } from 'crypto';

import { ExperimentsItem } from '@suite-common/suite-types';

import { ExperimentIdType } from './experiments';

export type ExperimentsItemUuidType = Omit<ExperimentsItem, 'id'> & { id: ExperimentIdType };

type ExperimentCategoriesProps = {
experiment: ExperimentsItemUuidType | undefined;
instanceId: string | undefined;
};

type ExperimentsGroupsType = ExperimentsItemUuidType['groups'];
type ExperimentsGroupType = ExperimentsGroupsType[number];

type ExperimentGetGroupByInclusion = {
groups: ExperimentsGroupsType;
inclusion: number;
};

/**
* @returns number between 0 and 99 generated from instanceId and experimentId
*/
export const getInclusionFromInstanceId = (instanceId: string, experimentId: ExperimentIdType) => {
const combinedId = `${instanceId}-${experimentId}`;
const hash = createHash('sha256').update(combinedId).digest('hex').slice(0, 8);

return parseInt(hash, 16) % 100;
};

export const getExperimentGroupByInclusion = ({
groups,
inclusion,
}: ExperimentGetGroupByInclusion): ExperimentsGroupType | undefined => {
let currentPercentage = 0;

const extendedExperiment = groups.map(group => {
const result = {
group,
range: [currentPercentage, currentPercentage + group.percentage - 1],
};

currentPercentage += group.percentage;

return result;
});

return extendedExperiment.find(
group => group.range[0] <= inclusion && group.range[1] >= inclusion,
)?.group;
};

export const selectActiveExperimentGroup = ({
experiment,
instanceId,
}: ExperimentCategoriesProps): ExperimentsGroupType | undefined => {
if (!instanceId || !experiment) return undefined;

const inclusionFromInstanceId = getInclusionFromInstanceId(instanceId, experiment.id);
const { groups } = experiment;

const experimentRange = getExperimentGroupByInclusion({
groups,
inclusion: inclusionFromInstanceId,
});

return experimentRange;
};
3 changes: 3 additions & 0 deletions suite-common/message-system/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,6 @@ export * from './messageSystemSelectors';
export * from './messageSystemThunks';
export * from './messageSystemTypes';
export * from './messageSystemUtils';

export * from './experiment';
export * from './experiment/experiments';
8 changes: 6 additions & 2 deletions suite-common/message-system/src/messageSystemSelectors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import { createWeakMapSelector, returnStableArrayIfEmpty } from '@suite-common/r
import { Message, Category } from '@suite-common/suite-types';

import { ContextDomain, FeatureDomain, MessageSystemRootState } from './messageSystemTypes';
import { ExperimentIdType } from './experiment/experiments';
import { ExperimentsItemUuidType } from './experiment';

// Create app-specific selectors with correct types
export const createMemoizedSelector = createWeakMapSelector.withTypes<MessageSystemRootState>();
Expand Down Expand Up @@ -165,7 +167,9 @@ export const selectAllValidExperiments = createMemoizedSelector(
},
);

export const selectExperimentById = (id: string) =>
export const selectExperimentById = (id: ExperimentIdType) =>
createMemoizedSelector([selectAllValidExperiments], allValidExperiments =>
allValidExperiments.find(experiment => experiment.id === id),
allValidExperiments.find(
(experiment): experiment is ExperimentsItemUuidType => experiment.id === id,
),
);

0 comments on commit fc56ea0

Please sign in to comment.