Skip to content

Commit 8ad86bf

Browse files
ASaiAnudeepASaiAnudeep
authored
feat: failure analysis (#251)
1 parent 6a38155 commit 8ad86bf

11 files changed

+235
-29
lines changed

‎.vscode/settings.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"cSpell.words": [
3+
"analysed",
34
"mstest",
45
"nunit",
56
"pactum",

‎src/beats/beats.js

+33-18
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
const { getCIInformation } = require('../helpers/ci');
22
const logger = require('../utils/logger');
33
const { BeatsApi } = require('./beats.api');
4-
const { HOOK } = require('../helpers/constants');
4+
const { HOOK, PROCESS_STATUS } = require('../helpers/constants');
55
const TestResult = require('test-results-parser/src/models/TestResult');
66
const { BeatsAttachments } = require('./beats.attachments');
77

@@ -31,9 +31,7 @@ class Beats {
3131
await this.#publishTestResults();
3232
await this.#uploadAttachments();
3333
this.#updateTitleLink();
34-
await this.#attachFailureSummary();
35-
await this.#attachSmartAnalysis();
36-
await this.#attachErrorClusters();
34+
await this.#attachExtensions();
3735
}
3836

3937
#setCIInfo() {
@@ -104,13 +102,20 @@ class Beats {
104102
}
105103
}
106104

107-
async #attachFailureSummary() {
105+
async #attachExtensions() {
108106
if (!this.test_run_id) {
109107
return;
110108
}
111109
if (!this.config.targets) {
112110
return;
113111
}
112+
await this.#attachFailureSummary();
113+
await this.#attachFailureAnalysis();
114+
await this.#attachSmartAnalysis();
115+
await this.#attachErrorClusters();
116+
}
117+
118+
async #attachFailureSummary() {
114119
if (this.result.status !== 'FAIL') {
115120
return;
116121
}
@@ -132,13 +137,29 @@ class Beats {
132137
}
133138
}
134139

135-
async #attachSmartAnalysis() {
136-
if (!this.test_run_id) {
140+
async #attachFailureAnalysis() {
141+
if (this.result.status !== 'FAIL') {
137142
return;
138143
}
139-
if (!this.config.targets) {
144+
if (this.config.show_failure_analysis === false) {
140145
return;
141146
}
147+
try {
148+
logger.info('🪄 Fetching Failure Analysis...');
149+
await this.#setTestRun('Failure Analysis Status', 'failure_analysis_status');
150+
this.config.extensions.push({
151+
name: 'failure-analysis',
152+
hook: HOOK.AFTER_SUMMARY,
153+
inputs: {
154+
data: this.test_run
155+
}
156+
});
157+
} catch (error) {
158+
logger.error(`❌ Unable to attach failure analysis: ${error.message}`, error);
159+
}
160+
}
161+
162+
async #attachSmartAnalysis() {
142163
if (this.config.show_smart_analysis === false) {
143164
return;
144165
}
@@ -165,7 +186,7 @@ class Beats {
165186
}
166187

167188
async #setTestRun(text, wait_for = 'smart_analysis_status') {
168-
if (this.test_run && this.test_run[wait_for] === 'COMPLETED') {
189+
if (this.test_run && this.test_run[wait_for] === PROCESS_STATUS.COMPLETED) {
169190
return;
170191
}
171192
let retry = 3;
@@ -175,13 +196,13 @@ class Beats {
175196
this.test_run = await this.api.getTestRun(this.test_run_id);
176197
const status = this.test_run && this.test_run[wait_for];
177198
switch (status) {
178-
case 'COMPLETED':
199+
case PROCESS_STATUS.COMPLETED:
179200
logger.debug(`☑️ ${text} generated successfully`);
180201
return;
181-
case 'FAILED':
202+
case PROCESS_STATUS.FAILED:
182203
logger.error(`❌ Failed to generate ${text}`);
183204
return;
184-
case 'SKIPPED':
205+
case PROCESS_STATUS.SKIPPED:
185206
logger.warn(`❗ Skipped generating ${text}`);
186207
return;
187208
}
@@ -191,12 +212,6 @@ class Beats {
191212
}
192213

193214
async #attachErrorClusters() {
194-
if (!this.test_run_id) {
195-
return;
196-
}
197-
if (!this.config.targets) {
198-
return;
199-
}
200215
if (this.result.status !== 'FAIL') {
201216
return;
202217
}

‎src/beats/beats.types.d.ts

+6
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@ export type IBeatExecutionMetric = {
99
added: number
1010
removed: number
1111
flaky: number
12+
product_bugs: number
13+
environment_issues: number
14+
automation_bugs: number
15+
not_a_defects: number
16+
to_investigate: number
17+
auto_analysed: number
1218
failure_summary: any
1319
failure_summary_provider: any
1420
failure_summary_model: any

‎src/extensions/failure-analysis.extension.js

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
const { BaseExtension } = require('./base.extension');
2+
const { STATUS, HOOK } = require("../helpers/constants");
3+
4+
class FailureAnalysisExtension extends BaseExtension {
5+
6+
constructor(target, extension, result, payload, root_payload) {
7+
super(target, extension, result, payload, root_payload);
8+
this.#setDefaultOptions();
9+
this.#setDefaultInputs();
10+
this.updateExtensionInputs();
11+
}
12+
13+
#setDefaultOptions() {
14+
this.default_options.hook = HOOK.AFTER_SUMMARY,
15+
this.default_options.condition = STATUS.PASS_OR_FAIL;
16+
}
17+
18+
#setDefaultInputs() {
19+
this.default_inputs.title = '';
20+
this.default_inputs.title_link = '';
21+
}
22+
23+
run() {
24+
this.#setText();
25+
this.attach();
26+
}
27+
28+
#setText() {
29+
const data = this.extension.inputs.data;
30+
if (!data) {
31+
return;
32+
}
33+
34+
/**
35+
* @type {import('../beats/beats.types').IBeatExecutionMetric}
36+
*/
37+
const execution_metrics = data.execution_metrics[0];
38+
39+
if (!execution_metrics) {
40+
logger.warn('⚠️ No execution metrics found. Skipping.');
41+
return;
42+
}
43+
44+
const failure_analysis = [];
45+
46+
if (execution_metrics.to_investigate) {
47+
failure_analysis.push(`🔎 To Investigate: ${execution_metrics.to_investigate}`);
48+
}
49+
if (execution_metrics.auto_analysed) {
50+
failure_analysis.push(`🪄 Auto Analysed: ${execution_metrics.auto_analysed}`);
51+
}
52+
53+
this.text = failure_analysis.join('  •  ');
54+
}
55+
56+
}
57+
58+
module.exports = { FailureAnalysisExtension };

‎src/extensions/index.js

+3
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ const { EXTENSION } = require('../helpers/constants');
1313
const { checkCondition } = require('../helpers/helper');
1414
const logger = require('../utils/logger');
1515
const { ErrorClustersExtension } = require('./error-clusters.extension');
16+
const { FailureAnalysisExtension } = require('./failure-analysis.extension');
1617

1718
async function run(options) {
1819
const { target, result, hook } = options;
@@ -59,6 +60,8 @@ function getExtensionRunner(extension, options) {
5960
return new CIInfoExtension(options.target, extension, options.result, options.payload, options.root_payload);
6061
case EXTENSION.AI_FAILURE_SUMMARY:
6162
return new AIFailureSummaryExtension(options.target, extension, options.result, options.payload, options.root_payload);
63+
case EXTENSION.FAILURE_ANALYSIS:
64+
return new FailureAnalysisExtension(options.target, extension, options.result, options.payload, options.root_payload);
6265
case EXTENSION.SMART_ANALYSIS:
6366
return new SmartAnalysisExtension(options.target, extension, options.result, options.payload, options.root_payload);
6467
case EXTENSION.ERROR_CLUSTERS:

‎src/extensions/smart-analysis.extension.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -65,13 +65,13 @@ class SmartAnalysisExtension extends BaseExtension {
6565
for (const item of smart_analysis) {
6666
rows.push(item);
6767
if (rows.length === 3) {
68-
texts.push(rows.join(' ｜ '));
68+
texts.push(rows.join('  •  '));
6969
rows.length = 0;
7070
}
7171
}
7272

7373
if (rows.length > 0) {
74-
texts.push(rows.join(' ｜ '));
74+
texts.push(rows.join('  •  '));
7575
}
7676

7777
this.text = this.mergeTexts(texts);

‎src/helpers/constants.js

+9
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ const TARGET = Object.freeze({
2121

2222
const EXTENSION = Object.freeze({
2323
AI_FAILURE_SUMMARY: 'ai-failure-summary',
24+
FAILURE_ANALYSIS: 'failure-analysis',
2425
SMART_ANALYSIS: 'smart-analysis',
2526
ERROR_CLUSTERS: 'error-clusters',
2627
HYPERLINKS: 'hyperlinks',
@@ -39,6 +40,13 @@ const URLS = Object.freeze({
3940
QUICK_CHART: 'https://quickchart.io'
4041
});
4142

43+
const PROCESS_STATUS = Object.freeze({
44+
RUNNING: 'RUNNING',
45+
COMPLETED: 'COMPLETED',
46+
FAILED: 'FAILED',
47+
SKIPPED: 'SKIPPED',
48+
});
49+
4250
const MIN_NODE_VERSION = 14;
4351

4452
module.exports = Object.freeze({
@@ -47,5 +55,6 @@ module.exports = Object.freeze({
4755
TARGET,
4856
EXTENSION,
4957
URLS,
58+
PROCESS_STATUS,
5059
MIN_NODE_VERSION
5160
});

‎src/index.d.ts

+1
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,7 @@ export interface PublishReport {
229229
project?: string;
230230
run?: string;
231231
show_failure_summary?: boolean;
232+
show_failure_analysis?: boolean;
232233
show_smart_analysis?: boolean;
233234
show_error_clusters?: boolean;
234235
targets?: Target[];

‎test/beats.spec.js

+34
Original file line numberDiff line numberDiff line change
@@ -230,4 +230,38 @@ describe('TestBeats', () => {
230230
assert.equal(mock.getInteraction(id4).exercised, true);
231231
});
232232

233+
it('should send results with failure analysis to beats', async () => {
234+
const id1 = mock.addInteraction('post test results to beats');
235+
const id2 = mock.addInteraction('get test results with failure analysis from beats');
236+
const id3 = mock.addInteraction('get empty error clusters from beats');
237+
const id4 = mock.addInteraction('post test-summary with beats to teams with ai failure summary and smart analysis and failure analysis');
238+
await publish({
239+
config: {
240+
api_key: 'api-key',
241+
project: 'project-name',
242+
run: 'build-name',
243+
targets: [
244+
{
245+
name: 'teams',
246+
inputs: {
247+
url: 'http://localhost:9393/message'
248+
}
249+
}
250+
],
251+
results: [
252+
{
253+
type: 'testng',
254+
files: [
255+
'test/data/testng/single-suite-failures.xml'
256+
]
257+
}
258+
]
259+
}
260+
});
261+
assert.equal(mock.getInteraction(id1).exercised, true);
262+
assert.equal(mock.getInteraction(id2).exercised, true);
263+
assert.equal(mock.getInteraction(id3).exercised, true);
264+
assert.equal(mock.getInteraction(id4).exercised, true);
265+
});
266+
233267
});

‎test/mocks/beats.mock.js

+37
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ addInteractionHandler('get test results from beats', () => {
2929
body: {
3030
id: 'test-run-id',
3131
"failure_summary_status": "COMPLETED",
32+
"failure_analysis_status": "COMPLETED",
3233
"smart_analysis_status": "COMPLETED",
3334
"execution_metrics": [
3435
{
@@ -69,6 +70,42 @@ addInteractionHandler('get test results with smart analysis from beats', () => {
6970
}
7071
});
7172

73+
addInteractionHandler('get test results with failure analysis from beats', () => {
74+
return {
75+
strict: false,
76+
request: {
77+
method: 'GET',
78+
path: '/api/core/v1/test-runs/test-run-id'
79+
},
80+
response: {
81+
status: 200,
82+
body: {
83+
id: 'test-run-id',
84+
"failure_summary_status": "COMPLETED",
85+
"failure_analysis_status": "COMPLETED",
86+
"smart_analysis_status": "SKIPPED",
87+
"execution_metrics": [
88+
{
89+
"failure_summary": "",
90+
"newly_failed": 1,
91+
"always_failing": 1,
92+
"recovered": 1,
93+
"added": 0,
94+
"removed": 0,
95+
"flaky": 1,
96+
"product_bugs": 1,
97+
"environment_issues": 1,
98+
"automation_bugs": 1,
99+
"not_a_defects": 1,
100+
"to_investigate": 1,
101+
"auto_analysed": 1
102+
}
103+
]
104+
}
105+
}
106+
}
107+
});
108+
72109
addInteractionHandler('get error clusters from beats', () => {
73110
return {
74111
strict: false,

0 commit comments

Comments
 (0)