Skip to content

Commit

Permalink
Optional Slack notifications (#180)
Browse files Browse the repository at this point in the history
  • Loading branch information
ngmachado authored Mar 15, 2023
1 parent b8a1925 commit 66d00fb
Show file tree
Hide file tree
Showing 13 changed files with 272 additions and 93 deletions.
187 changes: 122 additions & 65 deletions package-lock.json

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
"license": "MIT",
"dependencies": {
"@decentral.ee/web3-helpers": "^0.5.3",
"@slack/webhook": "^6.1.0",
"@superfluid-finance/ethereum-contracts": "1.0.0",
"@superfluid-finance/js-sdk": "0.5.12",
"@superfluid-finance/metadata": "github:superfluid-finance/metadata#BatchLiquidatorV2",
Expand Down
22 changes: 22 additions & 0 deletions src/app.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,11 @@ const Repository = require("./database/repository");
const utils = require("./utils/utils.js");
const HTTPServer = require("./httpserver/server");
const Report = require("./httpserver/report");
const Notifier = require("./services/notifier");
const SlackNotifier = require("./services/slackNotifier");
const NotifierJobs = require("./services/notificationJobs");
const Errors = require("./utils/errors/errors");
const { wad4human } = require("@decentral.ee/web3-helpers");

class App {
/*
Expand Down Expand Up @@ -57,6 +61,13 @@ class App {
this.server = new HTTPServer(this);
this.timer = new Timer();

this.notifier = new Notifier(this);
// at this stage we only work with slack
if (this.config.SLACK_WEBHOOK_URL) {
this._slackNotifier = new SlackNotifier(this, {timeout: 3000});
this.notificationJobs = new NotifierJobs(this);
}

this._isShutdown = false;
this._isInitialized = false;
}
Expand Down Expand Up @@ -146,6 +157,8 @@ class App {
try {
this.logger.debug(`booting sentinel`);
this._isShutdown = false;
// send notification about time sentinel started including timestamp
this.notifier.sendNotification(`Sentinel started at ${new Date()}`);
// connect to provided rpc
await this.client.connect();
// if we are running tests don't try to load network information
Expand All @@ -154,6 +167,7 @@ class App {
}
// create all web3 infrastructure needed
await this.client.init();
this.notifier.sendNotification(`RPC connected with chainId ${await this.client.getChainId()}, account ${this.client.agentAccounts?.address} has balance ${this.client.agentAccounts ? wad4human(await this.client.getAccountBalance()) : "N/A"}`);
if (this.config.BATCH_CONTRACT !== undefined) {
await this.client.loadBatchContract();
}
Expand All @@ -180,6 +194,9 @@ class App {
this.logger.debug(JSON.stringify(userConfig));
if (await this.isResyncNeeded(userConfig)) {
this.logger.error(`ATTENTION: Configuration changed since last run, please re-sync.`);
// send notification about configuration change, and exit
this.notifier.sendNotification(`Configuration changed since last run, please re-sync.`);
await this.timer.timeout(3500);
process.exit(1);
}
await this.db.queries.saveConfiguration(JSON.stringify(userConfig));
Expand All @@ -194,6 +211,11 @@ class App {
if (this.config.METRICS === true) {
this.timer.startAfter(this.server);
}
// Only start notification jobs if notifier is enabled
if (this.notificationJobs) {
this.logger.info(`Starting notification jobs`);
this.timer.startAfter(this.notificationJobs);
}
//from this point on, sentinel is considered initialized.
this._isInitialized = true;
// await x milliseconds before running next liquidation job
Expand Down
2 changes: 1 addition & 1 deletion src/boot/loadEvents.js
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ class LoadEvents {
let keepTrying = 10;
while (true) {
try {
await task.self.app.protocol.calculateAndSaveTokenDelay(task.token);
await task.self.app.protocol.calculateAndSaveTokenDelay(task.token, false);
break;
} catch (err) {
keepTrying++;
Expand Down
4 changes: 3 additions & 1 deletion src/config/configuration.js
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ class Config {
this.FASTSYNC = this._parseToBool(process.env.FASTSYNC, true);
this.IPFS_GATEWAY = process.env.IPFS_GATEWAY || "https://cloudflare-ipfs.com/ipfs/";
this.PIRATE = this._parseToBool(process.env.PIRATE, false);
this.SLACK_WEBHOOK_URL = process.env.SLACK_WEBHOOK_URL;

// extra options: undoc and excluded from cmdline parser. Use .env file to change the defaults.
this.CONCURRENCY = process.env.CONCURRENCY || 1;
Expand Down Expand Up @@ -186,7 +187,8 @@ class Config {
LOG_LEVEL: this.LOG_LEVEL,
POLLING_INTERVAL: this.POLLING_INTERVAL,
BLOCK_OFFSET: this.BLOCK_OFFSET,
MAX_TX_NUMBER: this.MAX_TX_NUMBER
MAX_TX_NUMBER: this.MAX_TX_NUMBER,
SLACK_WEBHOOK_URL: this.SLACK_WEBHOOK_URL
};
}
}
Expand Down
59 changes: 35 additions & 24 deletions src/protocol/protocol.js
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@ class Protocol {
new BN(arrPromise[0]),
new BN(arrPromise[1].availableBalance),
new BN(arrPromise[1].deposit),
this.app.client.superTokens[token.toLowerCase()].liquidation_period,
this.app.client.superTokens[token.toLowerCase()].patrician_period
this.app.client.superTokens[token.toLowerCase()].liquidation_period,
this.app.client.superTokens[token.toLowerCase()].patrician_period
);
} catch (err) {
console.error(err);
Expand Down Expand Up @@ -120,32 +120,43 @@ class Protocol {
}
}

async calculateAndSaveTokenDelay (superToken) {
async calculateAndSaveTokenDelay (superToken, sendNotification = false) {
try {
if(!this.app.config.OBSERVER) {
const tokenInfo = this.app.client.superTokenNames[superToken.toLowerCase()];
const currentTokenPIC = await this.getCurrentPIC(superToken);
const rewardAccount = await this.getRewardAddress(superToken);
const token = await this.app.db.models.SuperTokenModel.findOne({ where: { address: this.app.client.web3.utils.toChecksumAddress(superToken) } });
token.pic = currentTokenPIC === undefined ? undefined : currentTokenPIC.pic;
token.pppmode = this.app.config.PIRATE ? this.PPPMode.Pirate : this.PPPMode.Pleb;

if (this.app.config.PIC === undefined) {
this.app.logger.debug(`${tokenInfo}: no PIC configured, default to ${this.app.config.PIRATE ? "Pirate" : "Pleb"}`);
} else if (currentTokenPIC !== undefined && this.app.config.PIC.toLowerCase() === currentTokenPIC.pic.toLowerCase()) {
token.pppmode = this.PPPMode.Patrician;
this.app.logger.info(`${tokenInfo}: PIC active`);
} else if (rewardAccount.toLowerCase() === this.app.config.PIC.toLowerCase()) {
token.pppmode = this.PPPMode.Patrician;
this.app.logger.debug(`${tokenInfo}: configured PIC match reward address directly, set as PIC`);
} else {
this.app.logger.debug(`${tokenInfo}: you are not the PIC, default to ${this.app.config.PIRATE ? "Pirate" : "Pleb"}`);
}
await token.save();
} else {

if(this.app.config.OBSERVER) {
this.app.logger.info("running as observer, ignoring PIC event");
return;
}

const tokenInfo = this.app.client.superTokenNames[superToken.toLowerCase()];
const currentTokenPIC = await this.getCurrentPIC(superToken);
const rewardAccount = await this.getRewardAddress(superToken);
const token = await this.app.db.models.SuperTokenModel.findOne({ where: { address: this.app.client.web3.utils.toChecksumAddress(superToken) } });
token.pic = currentTokenPIC === undefined ? undefined : currentTokenPIC.pic;
token.pppmode = this.app.config.PIRATE ? this.PPPMode.Pirate : this.PPPMode.Pleb;

let msg;
if (this.app.config.PIC === undefined) {
msg = `${tokenInfo}: no PIC configured, default to ${this.app.config.PIRATE ? "Pirate" : "Pleb"}`;
this.app.logger.debug(msg);
} else if (currentTokenPIC !== undefined && this.app.config.PIC.toLowerCase() === currentTokenPIC.pic.toLowerCase()) {
token.pppmode = this.PPPMode.Patrician;
msg = `${tokenInfo}: you are the active PIC now`;
this.app.logger.info(msg);
} else if (rewardAccount.toLowerCase() === this.app.config.PIC.toLowerCase()) {
token.pppmode = this.PPPMode.Patrician;
msg = `${tokenInfo}: your configured PIC matches the token's reward address (no TOGA set)`;
this.app.logger.debug(msg);
} else {
msg = `${tokenInfo}: you are not the PIC, default to ${this.app.config.PIRATE ? "Pirate" : "Pleb"}`;
this.app.logger.debug(msg);
}

if(sendNotification) {
this.app.notifier.sendNotification(msg);
}

await token.save();
} catch (err) {
this.app.logger.error(err);
throw Error(`Protocol.calculateAndSaveTokenDelay(): ${err}`);
Expand Down
37 changes: 37 additions & 0 deletions src/services/notificationJobs.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
const {wad4human} = require("@decentral.ee/web3-helpers/src/math-utils");

const timeout = ms => new Promise(resolve => setTimeout(resolve, ms));
async function trigger (obj, ms) {
await timeout(ms);
await obj.sendReport();
}

class NotificationJobs {
constructor(app) {
this.app = app;
}

async sendReport () {
const healthcheck = await this.app.healthReport.fullReport();
if(!healthcheck.healthy) {
const healthData = `Healthy: ${healthcheck.healthy}\nChainId: ${healthcheck.network.chainId}`;
this.app.notifier.sendNotification(healthData);
}
}

async start () {
// run every hour ( value in ms)
this.run(this, 3600*1000);
}

async run (self, time) {
if (self.app._isShutdown) {
self.app.logger.info(`app.shutdown() - closing NotificationJobs`);
return;
}
await trigger(self, time);
await this.run(self, time);
}
}

module.exports = NotificationJobs;
9 changes: 9 additions & 0 deletions src/services/notifier.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
const EventEmitter = require('events');

class Notifier extends EventEmitter {
sendNotification(message) {
this.emit('notification', `[${process.pid}]: ${message}`);
}
}

module.exports = Notifier;
38 changes: 38 additions & 0 deletions src/services/slackNotifier.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
const { IncomingWebhook } = require("@slack/webhook");

/*
Slack Notifier service
This is not called directly, but is used by the Notifier service
*/

class SlackNotifier {
constructor(app, options) {
if (!app.config.SLACK_WEBHOOK_URL) {
throw new Error('Slack webhook url must be set in config');
}

// notification service must be initialized
if (!app.notifier) {
throw new Error('Notifier must be initialized before SlackNotifier');
}

this.app = app;
this.webhook = new IncomingWebhook(app.config.SLACK_WEBHOOK_URL, options);
this.app.notifier.on('notification', message => {
this.sendNotification(message);
});
}

async sendNotification(message) {
try {
await this.webhook.send({
text: message,
});
this.app.logger.info(`SlackNotifier: Sent notification to Slack: ${message}`);
} catch (error) {
this.app.logger.error(`SlackNotifier: Error sending notification to Slack: ${error}`);
}
}
}

module.exports = SlackNotifier;
1 change: 1 addition & 0 deletions src/transaction/gas.js
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ class Gas {
parseInt(originalGasPrice) >= this.app.config.MAX_GAS_PRICE
) {
this.app.logger.debug(`Hit gas price limit of ${this.app.config.MAX_GAS_PRICE}`);
this.app.notifier.sendNotification(`Hit gas price limit of ${this.app.config.MAX_GAS_PRICE}`);
gasPrice = this.app.config.MAX_GAS_PRICE;
} else {
gasPrice = Math.ceil(parseInt(gasPrice) * step);
Expand Down
2 changes: 1 addition & 1 deletion src/web3client/client.js
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ class Client {
});
// use for runtime subscription
if(setPIC) {
this.app.protocol.calculateAndSaveTokenDelay(newSuperToken);
this.app.protocol.calculateAndSaveTokenDelay(newSuperToken, false);
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/web3client/eventTracker.js
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ class EventTracker {
if(event && event.eventName === "NewPIC") {
if (this.app.client.isSuperTokenRegistered(event.token)) {
this.app.logger.info(`[TOGA]: ${event.eventName} [${event.token}] new pic ${event.pic}`);
await this.app.protocol.calculateAndSaveTokenDelay(event.token);
await this.app.protocol.calculateAndSaveTokenDelay(event.token, true);
} else {
this.app.logger.debug(`[TOGA]: token ${event.token} is not subscribed`);
}
Expand Down
1 change: 1 addition & 0 deletions src/web3client/liquidator.js
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,7 @@ class Liquidator {
}
if(error instanceof this.app.Errors.AccountFundsError) {
this.app.logger.warn(`insufficient funds agent account`);
this.app.notifier.sendNotification(`Insufficient funds agent account to send tx ${signed.tx.transactionHash}`);
return {
error: error.message,
tx: undefined
Expand Down

0 comments on commit 66d00fb

Please sign in to comment.