From 00449b2df207ee0d5abc7f233fa7c611621605ac Mon Sep 17 00:00:00 2001 From: Mavrik Date: Fri, 20 Dec 2024 10:40:39 +0100 Subject: [PATCH 1/4] fix: add missing backend package --- backend/package.json | 1 + backend/yarn.lock | 6 ++++++ yarn.lock | 18 ++++++++++++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/backend/package.json b/backend/package.json index f93f14e5..2a911ac0 100644 --- a/backend/package.json +++ b/backend/package.json @@ -54,6 +54,7 @@ "@nestjs/schematics": "^10.0.0", "@nestjs/testing": "^10.0.0", "@types/cookie-parser": "^1.4.7", + "@types/eventsource": "^1.1.15", "@types/express": "^4.17.17", "@types/jest": "^29.5.2", "@types/node": "^20.3.1", diff --git a/backend/yarn.lock b/backend/yarn.lock index a2824a9a..3608ea1b 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -1068,6 +1068,11 @@ resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== +"@types/eventsource@^1.1.15": + version "1.1.15" + resolved "https://registry.yarnpkg.com/@types/eventsource/-/eventsource-1.1.15.tgz#949383d3482e20557cbecbf3b038368d94b6be27" + integrity sha512-XQmGcbnxUNa06HR3VBVkc9+A2Vpi9ZyLJcdS5dwaQQ/4ZMWFO+5c90FnMUpbtMZwB/FChoYHwuVg8TvkECacTA== + "@types/express-serve-static-core@^4.17.33": version "4.19.0" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.19.0.tgz#3ae8ab3767d98d0b682cda063c3339e1e86ccfaa" @@ -6049,6 +6054,7 @@ wkx@^0.5.0: "@types/node" "*" "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: + name wrap-ansi-cjs version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== diff --git a/yarn.lock b/yarn.lock index 355efca9..952cc6a5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2605,13 +2605,20 @@ resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.34.tgz#10964ba0dee6ac4cd462e2795b6bebd407303433" integrity sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g== -"@types/node@*", "@types/node@22.10.1": +"@types/node@*": version "22.10.1" resolved "https://registry.yarnpkg.com/@types/node/-/node-22.10.1.tgz#41ffeee127b8975a05f8c4f83fb89bcb2987d766" integrity sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ== dependencies: undici-types "~6.20.0" +"@types/node@22.10.2": + version "22.10.2" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.10.2.tgz#a485426e6d1fdafc7b0d4c7b24e2c78182ddabb9" + integrity sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ== + dependencies: + undici-types "~6.20.0" + "@types/node@22.7.5": version "22.7.5" resolved "https://registry.yarnpkg.com/@types/node/-/node-22.7.5.tgz#cfde981727a7ab3611a481510b473ae54442b92b" @@ -2641,7 +2648,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@18.3.12": +"@types/react@*": version "18.3.12" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.12.tgz#99419f182ccd69151813b7ee24b792fe08774f60" integrity sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw== @@ -2649,6 +2656,13 @@ "@types/prop-types" "*" csstype "^3.0.2" +"@types/react@19.0.2": + version "19.0.2" + resolved "https://registry.yarnpkg.com/@types/react/-/react-19.0.2.tgz#9363e6b3ef898c471cb182dd269decc4afc1b4f6" + integrity sha512-USU8ZI/xyKJwFTpjSVIrSeHBVAGagkHQKPNbxeWwql/vDmnTIBgx+TJnhFnj1NXgz8XfprU0egV2dROLGpsBEg== + dependencies: + csstype "^3.0.2" + "@types/stack-utils@^2.0.0": version "2.0.3" resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.3.tgz#6209321eb2c1712a7e7466422b8cb1fc0d9dd5d8" From 20ed3909a79ee4dd10a758ca2e4895f810847f00 Mon Sep 17 00:00:00 2001 From: Mavrik Date: Fri, 20 Dec 2024 10:42:48 +0100 Subject: [PATCH 2/4] feat: extract sse header data into constants --- backend/src/activity/activity.controller.ts | 10 +++------- src/constants/sse.ts | 8 ++++++++ 2 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 src/constants/sse.ts diff --git a/backend/src/activity/activity.controller.ts b/backend/src/activity/activity.controller.ts index 0eabbe8b..96c8157a 100644 --- a/backend/src/activity/activity.controller.ts +++ b/backend/src/activity/activity.controller.ts @@ -13,6 +13,7 @@ import { import { ActivityService } from './activity.service'; import { SessionGuard } from '../session.guard'; import { Request, Response } from 'express'; +import { KEEP_ALIVE_MESSAGE, SSE_HEADER } from '../../../src/constants/sse'; @Controller('activity') @UseGuards(SessionGuard) @@ -42,19 +43,14 @@ export class ActivityController { @Get('stream') sse(@Req() req: Request, @Res() res: Response) { - res.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'X-Accel-Buffering': 'no', - }); + res.writeHead(200, SSE_HEADER); res.flushHeaders(); this.activityService.addClient(res); const heartbeatInterval = setInterval(() => { - res.write(': keep-alive\n\n'); + res.write(KEEP_ALIVE_MESSAGE); }, 10000); req.on('close', () => { diff --git a/src/constants/sse.ts b/src/constants/sse.ts new file mode 100644 index 00000000..333c6895 --- /dev/null +++ b/src/constants/sse.ts @@ -0,0 +1,8 @@ +export const SSE_HEADER = { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'X-Accel-Buffering': 'no', +} + +export const KEEP_ALIVE_MESSAGE = ': keep-alive\n\n' From 198a20b948504d8876e41cb22d93fdd712870c90 Mon Sep 17 00:00:00 2001 From: Mavrik Date: Fri, 20 Dec 2024 10:43:29 +0100 Subject: [PATCH 3/4] feat: extract client logic into client manager util class --- backend/src/activity/activity.service.ts | 10 +++++----- backend/src/utils/client-manager.ts | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 backend/src/utils/client-manager.ts diff --git a/backend/src/activity/activity.service.ts b/backend/src/activity/activity.service.ts index e99f87b1..fb32558f 100644 --- a/backend/src/activity/activity.service.ts +++ b/backend/src/activity/activity.service.ts @@ -4,6 +4,7 @@ import { Activity } from './entities/activity.entity'; import { ActivityType } from '../../../src/types'; import { UpdateOptions, Op } from 'sequelize'; import { Response } from 'express'; +import { ClientManager } from '../utils/client-manager'; @Injectable() export class ActivityService { @@ -12,19 +13,18 @@ export class ActivityService { private activityRepository: typeof Activity, ) {} - private clients: Response[] = []; + private clientManager = new ClientManager(); public addClient(client: Response) { - this.clients.push(client); + this.clientManager.addClient(client); } public removeClient(client: Response) { - this.clients = this.clients.filter((c) => c !== client); + this.clientManager.removeClient(client); } public sendMessageToClients(data: any) { - const message = `data: ${JSON.stringify(data)}\n\n`; - this.clients.forEach((client) => client.write(message)); + this.clientManager.sendMessageToClients(data); } public async storeActivity(data: string, pubKey: string, type: ActivityType) { diff --git a/backend/src/utils/client-manager.ts b/backend/src/utils/client-manager.ts new file mode 100644 index 00000000..66aabaf3 --- /dev/null +++ b/backend/src/utils/client-manager.ts @@ -0,0 +1,18 @@ +import { Response } from 'express'; + +export class ClientManager { + private clients: Response[] = []; + + public addClient(client: Response): void { + this.clients.push(client); + } + + public removeClient(client: Response): void { + this.clients = this.clients.filter((c) => c !== client); + } + + public sendMessageToClients(data: any): void { + const message = `data: ${JSON.stringify(data)}\n\n`; + this.clients.forEach((client) => client.write(message)); + } +} From 1ba13432b84a6cc92359332469c57b56dccaf906 Mon Sep 17 00:00:00 2001 From: Mavrik Date: Fri, 20 Dec 2024 10:46:00 +0100 Subject: [PATCH 4/4] feat: add log sse service and endpoint --- backend/src/logs/logs.controller.ts | 21 ++++++++++++++++++++- backend/src/logs/logs.service.ts | 24 +++++++++++++++++++++--- package.json | 4 ++-- siren.js | 3 +++ src/components/AlertInfo/AlertInfo.tsx | 9 +++++++++ 5 files changed, 55 insertions(+), 6 deletions(-) diff --git a/backend/src/logs/logs.controller.ts b/backend/src/logs/logs.controller.ts index fc10ae85..5e573824 100644 --- a/backend/src/logs/logs.controller.ts +++ b/backend/src/logs/logs.controller.ts @@ -1,8 +1,8 @@ -// src/logs/logs.controller.ts import { Controller, Get, Res, Req, Param, UseGuards } from '@nestjs/common'; import { Request, Response } from 'express'; import { LogsService } from './logs.service'; import { SessionGuard } from '../session.guard'; +import { KEEP_ALIVE_MESSAGE, SSE_HEADER } from '../../../src/constants/sse'; @Controller('logs') @UseGuards(SessionGuard) @@ -26,6 +26,25 @@ export class LogsController { return this.logsService.readLogMetrics(); } + @Get('priority-log-stream') + sse(@Req() req: Request, @Res() res: Response) { + res.writeHead(200, SSE_HEADER); + + res.flushHeaders(); + + this.logsService.addClient(res); + + const heartbeatInterval = setInterval(() => { + res.write(KEEP_ALIVE_MESSAGE); + }, 10000); + + req.on('close', () => { + clearInterval(heartbeatInterval); + this.logsService.removeClient(res); + res.end(); + }); + } + @Get('dismiss/:index') dismissLogAlert(@Param('index') index: string) { return this.logsService.dismissLog(index); diff --git a/backend/src/logs/logs.service.ts b/backend/src/logs/logs.service.ts index d8ae12b8..7d48cb74 100644 --- a/backend/src/logs/logs.service.ts +++ b/backend/src/logs/logs.service.ts @@ -6,6 +6,7 @@ import { LogLevels, LogType, SSELog } from '../../../src/types'; import { InjectModel } from '@nestjs/sequelize'; import { Log } from './entities/log.entity'; import { Op } from 'sequelize'; +import { ClientManager } from '../utils/client-manager'; @Injectable() export class LogsService { @@ -20,6 +21,20 @@ export class LogsService { private sseStreams: Map> = new Map(); + private clientManager = new ClientManager(); + + public addClient(client: Response) { + this.clientManager.addClient(client); + } + + public removeClient(client: Response) { + this.clientManager.removeClient(client); + } + + public sendMessageToClients(data: any) { + this.clientManager.sendMessageToClients(data); + } + public async startSse(url: string, type: LogType) { console.log(`starting sse ${url}, ${type}...`); const eventSource = new EventSource(url); @@ -27,7 +42,7 @@ export class LogsService { const sseStream: Subject = new Subject(); this.sseStreams.set(url, sseStream); - eventSource.onmessage = (event) => { + eventSource.onmessage = async (event) => { let newData; try { @@ -39,10 +54,13 @@ export class LogsService { const { level } = newData; if (level !== LogLevels.INFO) { - this.logRepository.create( + const result = (await this.logRepository.create( { type, level, data: JSON.stringify(newData), isHidden: false }, { ignoreDuplicates: true }, - ); + )) as any; + + this.sendMessageToClients(result.dataValues); + if (this.isDebug) { console.log( newData, diff --git a/package.json b/package.json index f544d274..0b251bf9 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "@types/jest": "^29.5.12", "@types/js-cookie": "^3.0.6", "@types/jsonwebtoken": "^9.0.6", - "@types/react": "18.3.12", + "@types/react": "19.0.2", "@uiw/react-textarea-code-editor": "^2.1.1", "autoprefixer": "^10.4.19", "axios": "^0.27.2", @@ -85,7 +85,7 @@ ] }, "devDependencies": { - "@types/node": "22.10.1", + "@types/node": "22.10.2", "@typescript-eslint/eslint-plugin": "^7.1.0", "eslint": "8.57.0", "eslint-config-next": "^15.0.3", diff --git a/siren.js b/siren.js index b67de0b3..5809c10f 100644 --- a/siren.js +++ b/siren.js @@ -62,6 +62,9 @@ const handleSSe = (res, req, url) => { app.prepare().then(() => { const server = express() server.get('/activity-stream', (req, res) => handleSSe(res, req, `${backendUrl}/activity/stream`)) + server.get('/priority-log-stream', (req, res) => + handleSSe(res, req, `${backendUrl}/logs/priority-log-stream`), + ) server.get('/validator-logs', (req, res) => handleSSe(res, req, `${backendUrl}/logs/validator`)) server.get('/beacon-logs', (req, res) => handleSSe(res, req, `${backendUrl}/logs/beacon`)) diff --git a/src/components/AlertInfo/AlertInfo.tsx b/src/components/AlertInfo/AlertInfo.tsx index 585440e5..f8f140ea 100644 --- a/src/components/AlertInfo/AlertInfo.tsx +++ b/src/components/AlertInfo/AlertInfo.tsx @@ -5,6 +5,7 @@ import sortAlertMessagesBySeverity from '../../../utilities/sortAlerts' import useDiagnosticAlerts from '../../hooks/useDiagnosticAlerts' import useDivDimensions from '../../hooks/useDivDimensions' import useMediaQuery from '../../hooks/useMediaQuery' +import useSSEData from '../../hooks/useSSEData' import { proposerDuties } from '../../recoil/atoms' import { LogLevels, StatusColor } from '../../types' import AlertCard from '../AlertCard/AlertCard' @@ -24,6 +25,14 @@ const AlertInfo: FC = ({ metrics, ...props }) => { const [filter, setFilter] = useState('all') const duties = useRecoilValue(proposerDuties) + const { data: streamedData } = useSSEData({ + url: '/priority-log-stream', + isReady: true, + isStateStore: true, + }) + + console.log(streamedData) + const priorityLogAlerts = useMemo(() => { return Object.values(metrics) .flat()